8000 minimap2/index.c at v2.3 · Junaiid-Ahmad/minimap2 · GitHub
[go: up one dir, main page]

Skip to content

Latest commit

 

History

History
494 lines (455 loc) · 13.8 KB

index.c

File metadata and controls

494 lines (455 loc) · 13.8 KB
for (j = 0; j < seq->len; ++j) { // TODO: this is not the fastest way, but let's first see if speed matters here
uint64_t o = p->sum_len + j;
int c = seq_nt4_table[(uint8_t)s->seq[i].seq[j]];
mm_seq4_set(p->mi->S, o, c);
}
// update p->sum_len and p->mi->n_seq
p->sum_len += seq->len;
s->seq[i].rid = p->mi->n_seq++;
}
return s;
} else free(s);
} else if (step == 1) { // step 1: compute sketch
step_t *s = (step_t*)in;
for (i = 0; i < s->n_seq; ++i) {
mm_bseq1_t *t = &s->seq[i];
mm_sketch(0, t->seq, t->l_seq, p->mi->w, p->mi->k, t->rid, p->mi->is_hpc, &s->a);
free(t->seq); free(t->name);
}
free(s->seq); s->seq = 0;
return s;
} else if (step == 2) { // dispatch sketch to buckets
step_t *s = (step_t*)in;
mm_idx_add(p->mi, s->a.n, s->a.a);
kfree(0, s->a.a); free(s);
}
return 0;
}
mm_idx_t *mm_idx_gen(mm_bseq_file_t *fp, int w, int k, int b, int is_hpc, int mini_batch_size, int n_threads, uint64_t batch_size, int keep_name)
{
pipeline_t pl;
if (fp == 0 || mm_bseq_eof(fp)) return 0;
memset(&pl, 0, sizeof(pipeline_t));
pl.mini_batch_size = mini_batch_size < batch_size? mini_batch_size : batch_size;
pl.keep_name = keep_name;
pl.batch_size = batch_size;
pl.fp = fp;
pl.mi = mm_idx_init(w, k, b, is_hpc);
kt_pipeline(n_threads < 3? n_threads : 3, worker_pipeline, &pl, 3);
if (mm_verbose >= 3)
fprintf(stderr, "[M::%s::%.3f*%.2f] collected minimizers\n", __func__, realtime() - mm_realtime0, cputime() / (realtime() - mm_realtime0));
mm_idx_post(pl.mi, n_threads);
if (mm_verbose >= 3)
fprintf(stderr, "[M::%s::%.3f*%.2f] sorted minimizers\n", __func__, realtime() - mm_realtime0, cputime() / (realtime() - mm_realtime0));
return pl.mi;
}
mm_idx_t *mm_idx_build(const char *fn, int w, int k, int is_hpc, int n_threads) // a simpler interface
{
mm_bseq_file_t *fp;
mm_idx_t *mi;
fp = mm_bseq_open(fn);
if (fp == 0) return 0;
mi = mm_idx_gen(fp, w, k, 14, is_hpc, 1<<18, n_threads, UINT64_MAX, 1);
mm_bseq_close(fp);
return mi;
}
/*************
* index I/O *
*************/
void mm_idx_dump(FILE *fp, const mm_idx_t *mi)
{
uint64_t sum_len = 0;
uint32_t x[5];
int i;
x[0] = mi->w, x[1] = mi->k, x[2] = mi->b, x[3] = mi->n_seq, x[4] = mi->is_hpc;
fwrite(MM_IDX_MAGIC, 1, 4, fp);
fwrite(x, 4, 5, fp);
for (i = 0; i < mi->n_seq; ++i) {
uint8_t l;
l = strlen(mi->seq[i].name);
fwrite(&l, 1, 1, fp);
fwrite(mi->seq[i].name, 1, l, fp);
fwrite(&mi->seq[i].len, 4, 1, fp);
sum_len += mi->seq[i].len;
}
for (i = 0; i < 1<<mi->b; ++i) {
mm_idx_bucket_t *b = &mi->B[i];
khint_t k;
idxhash_t *h = (idxhash_t*)b->h;
uint32_t size = h? h->size : 0;
fwrite(&b->n, 4, 1, fp);
fwrite(b->p, 8, b->n, fp);
fwrite(&size, 4, 1, fp);
if (size == 0) continue;
for (k = 0; k < kh_end(h); ++k) {
uint64_t x[2];
if (!kh_exist(h, k)) continue;
x[0] = kh_key(h, k), x[1] = kh_val(h, k);
fwrite(x, 8, 2, fp);
}
}
fwrite(mi->S, 4, (sum_len + 7) / 8, fp);
fflush(fp);
}
mm_idx_t *mm_idx_load(FILE *fp)
{
int i;
char magic[4];
uint32_t x[5];
uint64_t sum_len = 0;
mm_idx_t *mi;
if (fread(magic, 1, 4, fp) != 4) return 0;
if (strncmp(magic, MM_IDX_MAGIC, 4) != 0) return 0;
if (fread(x, 4, 5, fp) != 5) return 0;
mi = mm_idx_init(x[0], x[1], x[2], x[4]);
mi->n_seq = x[3];
mi->seq = (mm_idx_seq_t*)kcalloc(mi->km, mi->n_seq, sizeof(mm_idx_seq_t));
for (i = 0; i < mi->n_seq; ++i) {
uint8_t l;
mm_idx_seq_t *s = &mi->seq[i];
fread(&l, 1, 1, fp);
s->name = (char*)kmalloc(mi->km, l + 1);
fread(s->name, 1, l, fp);
s->name[l] = 0;
fread(&s->len, 4, 1, fp);
s->offset = sum_len;
sum_len += s->len;
}
for (i = 0; i < 1<<mi->b; ++i) {
mm_idx_bucket_t *b = &mi->B[i];
uint32_t j, size;
khint_t k;
idxhash_t *h;
fread(&b->n, 4, 1, fp);
b->p = (uint64_t*)malloc(b->n * 8);
fread(b->p, 8, b->n, fp);
fread(&size, 4, 1, fp);
if (size == 0) continue;
b->h = h = kh_init(idx);
kh_resize(idx, h, size);
for (j = 0; j < size; ++j) {
uint64_t x[2];
int absent;
fread(x, 8, 2, fp);
k = kh_put(idx, h, x[0], &absent);
assert(absent);
kh_val(h, k) = x[1];
}
}
mi->S = (uint32_t*)malloc((sum_len + 7) / 8 * 4);
fread(mi->S, 4, (sum_len + 7) / 8, fp);
return mi;
}
int64_t mm_idx_is_idx(const char *fn)
{
int fd, is_idx = 0;
off_t ret, off_end;
char magic[4];
if (strcmp(fn, "-") == 0) return 0; // read from pipe; not an index
fd = open(fn, O_RDONLY);
if (fd < 0) return -1; // error
if ((off_end = lseek(fd, 0, SEEK_END)) >= 4) {
lseek(fd, 0, SEEK_SET);
ret = read(fd, magic, 4);
if (ret == 4 && strncmp(magic, MM_IDX_MAGIC, 4) == 0)
is_idx = 1;
}
close(fd);
return is_idx? off_end : 0;
}
mm_idx_reader_t *mm_idx_reader_open(const char *fn, const mm_idxopt_t *opt, const char *fn_out)
{
int64_t is_idx;
mm_idx_reader_t *r;
is_idx = mm_idx_is_idx(fn);
if (is_idx < 0) return 0; // failed to open the index
r = (mm_idx_reader_t*)calloc(1, sizeof(mm_idx_reader_t));
r->is_idx = is_idx;
if (opt) r->opt = *opt;
else mm_idxopt_init(&r->opt);
if (r->is_idx) {
r->fp.idx = fopen(fn, "rb");
r->idx_size = is_idx;
} else r->fp.seq = mm_bseq_open(fn);
if (fn_out) r->fp_out = fopen(fn_out, "wb");
return r;
}
void mm_idx_reader_close(mm_idx_reader_t *r)
{
if (r->is_idx) fclose(r->fp.idx);
else mm_bseq_close(r->fp.seq);
if (r->fp_out) fclose(r->fp_out);
free(r);
}
mm_idx_t *mm_idx_reader_read(mm_idx_reader_t *r, int n_threads)
{
mm_idx_t *mi;
if (r->is_idx) {
mi = mm_idx_load(r->fp.idx);
if (mi && mm_verbose >= 2 && (mi->k != r->opt.k || mi->w != r->opt.w || mi->is_hpc != r->opt.is_hpc))
fprintf(stderr, "[WARNING]\033[1;31m Indexing parameters (-k, -w or -H) overridden by parameters used in the prebuilt index.\033[0m\n");
} else
mi = mm_idx_gen(r->fp.seq, r->opt.w, r->opt.k, r->opt.bucket_bits, r->opt.is_hpc, r->opt.mini_batch_size, n_threads, r->opt.batch_size, 1);
if (mi) {
if (r->fp_out) mm_idx_dump(r->fp_out, mi);
++r->n_parts;
}
return mi;
}
int mm_idx_reader_eof(const mm_idx_reader_t *r) // TODO: in extremely rare cases, mm_bseq_eof() might not work
{
return r->is_idx? (feof(r->fp.idx) || ftell(r->fp.idx) == r->idx_size) : mm_bseq_eof(r->fp.seq);
}
0