razor.c
author Dan Winship <danw@gnome.org>
Thu Feb 07 09:58:48 2008 -0500 (2008-02-07)
changeset 115 26edeea5c95a
parent 114 1d54922ff920
child 116 4ec6e2a55c34
permissions -rw-r--r--
split array and hashtable code out into a new file
     1 #define _GNU_SOURCE
     2 
     3 #include <stdlib.h>
     4 #include <stddef.h>
     5 #include <stdint.h>
     6 #include <stdio.h>
     7 #include <string.h>
     8 #include <sys/types.h>
     9 #include <sys/stat.h>
    10 #include <sys/mman.h>
    11 #include <unistd.h>
    12 #include <fcntl.h>
    13 #include <errno.h>
    14 #include <ctype.h>
    15 #include <fnmatch.h>
    16 
    17 #include "razor.h"
    18 #include "razor-internal.h"
    19 #include "types.h"
    20 
    21 struct razor_set_section {
    22 	uint32_t type;
    23 	uint32_t offset;
    24 	uint32_t size;
    25 };
    26 
    27 struct razor_set_header {
    28 	uint32_t magic;
    29 	uint32_t version;
    30 	struct razor_set_section sections[0];
    31 };
    32 
    33 #define RAZOR_MAGIC 0x7a7a7a7a
    34 #define RAZOR_VERSION 1
    35 
    36 #define RAZOR_ENTRY_LAST	0x80000000ul
    37 #define RAZOR_IMMEDIATE		0x80000000ul
    38 #define RAZOR_ENTRY_MASK	0x00fffffful
    39 
    40 #define RAZOR_STRING_POOL	0
    41 #define RAZOR_PACKAGES		1
    42 #define RAZOR_PROPERTIES	2
    43 #define RAZOR_FILES		3
    44 #define RAZOR_PACKAGE_POOL	4
    45 #define RAZOR_PROPERTY_POOL	5
    46 #define RAZOR_FILE_POOL		6
    47 
    48 struct razor_package {
    49 	uint32_t name;
    50 	uint32_t version;
    51 	uint32_t properties;
    52 	uint32_t files;
    53 };
    54 
    55 struct razor_property {
    56 	uint32_t name;
    57 	uint32_t relation;
    58 	uint32_t version;
    59 	uint32_t packages;
    60 };
    61 
    62 struct razor_entry {
    63 	uint32_t name;
    64 	uint32_t start;
    65 	uint32_t packages;
    66 };
    67 
    68 struct razor_set {
    69 	struct array string_pool;
    70  	struct array packages;
    71  	struct array properties;
    72  	struct array files;
    73 	struct array package_pool;
    74  	struct array property_pool;
    75  	struct array file_pool;
    76 	struct razor_set_header *header;
    77 };
    78 
    79 struct import_entry {
    80 	uint32_t package;
    81 	char *name;
    82 };
    83 
    84 struct import_directory {
    85 	uint32_t name, count;
    86 	struct array files;
    87 	struct array packages;
    88 	struct import_directory *last;
    89 };
    90 
    91 struct razor_importer {
    92 	struct razor_set *set;
    93 	struct hashtable table;
    94 	struct razor_package *package;
    95 	struct array properties;
    96 	struct array files;
    97 };
    98 
    99 static void *
   100 zalloc(size_t size)
   101 {
   102 	void *p;
   103 
   104 	p = malloc(size);
   105 	memset(p, 0, size);
   106 
   107 	return p;
   108 }
   109 
   110 struct razor_set_section razor_sections[] = {
   111 	{ RAZOR_STRING_POOL,	offsetof(struct razor_set, string_pool) },
   112 	{ RAZOR_PACKAGES,	offsetof(struct razor_set, packages) },
   113 	{ RAZOR_PROPERTIES,	offsetof(struct razor_set, properties) },
   114 	{ RAZOR_FILES,		offsetof(struct razor_set, files) },
   115 	{ RAZOR_PACKAGE_POOL,	offsetof(struct razor_set, package_pool) },
   116 	{ RAZOR_PROPERTY_POOL,	offsetof(struct razor_set, property_pool) },
   117 	{ RAZOR_FILE_POOL,	offsetof(struct razor_set, file_pool) },
   118 };
   119 
   120 struct razor_set *
   121 razor_set_create(void)
   122 {
   123 	return zalloc(sizeof(struct razor_set));
   124 }
   125 
   126 struct razor_set *
   127 razor_set_open(const char *filename)
   128 {
   129 	struct razor_set *set;
   130 	struct razor_set_section *s;
   131 	struct stat stat;
   132 	struct array *array;
   133 	int fd;
   134 
   135 	set = zalloc(sizeof *set);
   136 	fd = open(filename, O_RDONLY);
   137 	if (fstat(fd, &stat) < 0)
   138 		return NULL;
   139 	set->header = mmap(NULL, stat.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
   140 	if (set->header == MAP_FAILED) {
   141 		free(set);
   142 		return NULL;
   143 	}
   144 
   145 	for (s = set->header->sections; ~s->type; s++) {
   146 		if (s->type >= ARRAY_SIZE(razor_sections))
   147 			continue;
   148 		if (s->type != razor_sections[s->type].type)
   149 			continue;
   150 		array = (void *) set + razor_sections[s->type].offset;
   151 		array->data = (void *) set->header + s->offset;
   152 		array->size = s->size;
   153 		array->alloc = s->size;
   154 	}
   155 	close(fd);
   156 
   157 	return set;
   158 }
   159 
   160 void
   161 razor_set_destroy(struct razor_set *set)
   162 {
   163 	unsigned int size;
   164 	struct array *a;
   165 	int i;
   166 
   167 	if (set->header) {
   168 		for (i = 0; set->header->sections[i].type; i++)
   169 			;
   170 		size = set->header->sections[i].type;
   171 		munmap(set->header, size);
   172 	} else {
   173 		for (i = 0; i < ARRAY_SIZE(razor_sections); i++) {
   174 			a = (void *) set + razor_sections[i].offset;
   175 			free(a->data);
   176 		}
   177 	}
   178 
   179 	free(set);
   180 }
   181 
   182 int
   183 razor_set_write(struct razor_set *set, const char *filename)
   184 {
   185 	char data[4096];
   186 	struct razor_set_header *header = (struct razor_set_header *) data;
   187 	struct array *a;
   188 	uint32_t offset;
   189 	int i, fd;
   190 
   191 	memset(data, 0, sizeof data);
   192 	header->magic = RAZOR_MAGIC;
   193 	header->version = RAZOR_VERSION;
   194 	offset = sizeof data;
   195 
   196 	for (i = 0; i < ARRAY_SIZE(razor_sections); i++) {
   197 		if (razor_sections[i].type != i)
   198 			continue;
   199 		a = (void *) set + razor_sections[i].offset;
   200 		header->sections[i].type = i;
   201 		header->sections[i].offset = offset;
   202 		header->sections[i].size = a->size;
   203 		offset += ALIGN(a->size, 4096);
   204 	}
   205 
   206 	header->sections[i].type = ~0;
   207 	header->sections[i].offset = 0;
   208 	header->sections[i].size = 0;
   209 
   210 	fd = open(filename, O_CREAT | O_WRONLY | O_TRUNC, 0666);
   211 	if (fd < 0)
   212 		return -1;
   213 
   214 	razor_write(fd, data, sizeof data);
   215 	memset(data, 0, sizeof data);
   216 	for (i = 0; i < ARRAY_SIZE(razor_sections); i++) {
   217 		if (razor_sections[i].type != i)
   218 			continue;
   219 		a = (void *) set + razor_sections[i].offset;
   220 		razor_write(fd, a->data, a->size);
   221 		razor_write(fd, data, ALIGN(a->size, 4096) - a->size);
   222 	}
   223 
   224 	close(fd);
   225 
   226 	return 0;
   227 }
   228 
   229 static uint32_t
   230 add_to_property_pool(struct array *pool, struct array *properties)
   231 {
   232 	uint32_t *p;
   233 
   234 	if (properties->size == 0)
   235 		return ~0;
   236 	else if (properties->size == sizeof *p)
   237 		return *(uint32_t *) properties->data | RAZOR_IMMEDIATE;
   238 
   239 	p = array_add(pool, properties->size);
   240 	memcpy(p, properties->data, properties->size);
   241 	p[properties->size / sizeof *p - 1] |= RAZOR_ENTRY_LAST;
   242 
   243 	return p - (uint32_t *) pool->data;
   244 }
   245 
   246 void
   247 razor_importer_begin_package(struct razor_importer *importer,
   248 			     const char *name, const char *version)
   249 {
   250 	struct razor_package *p;
   251 
   252 	p = array_add(&importer->set->packages, sizeof *p);
   253 	p->name = hashtable_tokenize(&importer->table, name);
   254 	p->version = hashtable_tokenize(&importer->table, version);
   255 
   256 	importer->package = p;
   257 	array_init(&importer->properties);
   258 }
   259 
   260 void
   261 razor_importer_finish_package(struct razor_importer *importer)
   262 {
   263 	importer->package->properties =
   264 		add_to_property_pool(&importer->set->property_pool,
   265 				     &importer->properties);
   266 
   267 	array_release(&importer->properties);
   268 }
   269 
   270 void
   271 razor_importer_add_property(struct razor_importer *importer,
   272 			    const char *name,
   273 			    enum razor_version_relation relation,
   274 			    const char *version,
   275 			    enum razor_property_type type)
   276 {
   277 	struct razor_property *p;
   278 	uint32_t *r;
   279 
   280 	p = array_add(&importer->set->properties, sizeof *p);
   281 	p->name = hashtable_tokenize(&importer->table, name) | (type << 30);
   282 	p->relation = relation;
   283 	p->version = hashtable_tokenize(&importer->table, version);
   284 	p->packages = importer->package -
   285 		(struct razor_package *) importer->set->packages.data;
   286 
   287 	r = array_add(&importer->properties, sizeof *r);
   288 	*r = p - (struct razor_property *) importer->set->properties.data;
   289 }
   290 
   291 void
   292 razor_importer_add_file(struct razor_importer *importer, const char *name)
   293 {
   294 	struct import_entry *e;
   295 
   296 	e = array_add(&importer->files, sizeof *e);
   297 
   298 	e->package = importer->package -
   299 		(struct razor_package *) importer->set->packages.data;
   300 	e->name = strdup(name);
   301 }
   302 
   303 struct razor_importer *
   304 razor_importer_new(void)
   305 {
   306 	struct razor_importer *importer;
   307 
   308 	importer = zalloc(sizeof *importer);
   309 	importer->set = razor_set_create();
   310 	hashtable_init(&importer->table, &importer->set->string_pool);
   311 
   312 	return importer;
   313 }
   314 
   315 /* Destroy an importer without creating the set. */
   316 void
   317 razor_importer_destroy(struct razor_importer *importer)
   318 {
   319 	/* FIXME: write this */
   320 }
   321 
   322 
   323 typedef int (*compare_with_data_func_t)(const void *p1,
   324 					const void *p,
   325 					void *data);
   326 
   327 struct qsort_context {
   328 	size_t size;
   329 	compare_with_data_func_t compare;
   330 	void *data;
   331 };
   332 
   333 static void
   334 qsort_swap(void *p1, void *p2, size_t size)
   335 {
   336 	char buffer[size];
   337 
   338 	memcpy(buffer, p1, size);
   339 	memcpy(p1, p2, size);
   340 	memcpy(p2, buffer, size);
   341 }
   342 
   343 static void
   344 __qsort_with_data(void *base, size_t nelem, uint32_t *map,
   345 		  struct qsort_context *ctx)
   346 {
   347 	void *p, *start, *end, *pivot;
   348 	uint32_t *mp, *mstart, *mend, tmp;
   349 	int left, right, result;
   350 	size_t size = ctx->size;
   351 
   352 	p = base;
   353 	start = base;
   354 	end = base + nelem * size;
   355 	mp = map;
   356 	mstart = map;
   357 	mend = map + nelem;
   358 	pivot = base + (random() % nelem) * size;
   359 
   360 	while (p < end) {
   361 		result = ctx->compare(p, pivot, ctx->data);
   362 		if (result < 0) {
   363 			qsort_swap(p, start, size);
   364 			tmp = *mp;
   365 			*mp = *mstart;
   366 			*mstart = tmp;
   367 			if (start == pivot)
   368 				pivot = p;
   369 			start += size;
   370 			mstart++;
   371 			p += size;
   372 			mp++;
   373 		} else if (result == 0) {
   374 			p += size;
   375 			mp++;
   376 		} else {
   377  			end -= size;
   378 			mend--;
   379 			qsort_swap(p, end, size);
   380 			tmp = *mp;
   381 			*mp = *mend;
   382 			*mend = tmp;
   383 			if (end == pivot)
   384 				pivot = p;
   385 		}
   386 	}
   387 
   388 	left = (start - base) / size;
   389 	right = (base + nelem * size - end) / size;
   390 	if (left > 1)
   391 		__qsort_with_data(base, left, map, ctx);
   392 	if (right > 1)
   393 		__qsort_with_data(end, right, mend, ctx);
   394 }
   395 
   396 uint32_t *
   397 qsort_with_data(void *base, size_t nelem, size_t size,
   398 		compare_with_data_func_t compare, void *data)
   399 {
   400 	struct qsort_context ctx;
   401 	uint32_t *map;
   402 	int i;
   403 
   404 	if (nelem == 0)
   405 		return NULL;
   406 
   407 	ctx.size = size;
   408 	ctx.compare = compare;
   409 	ctx.data = data;
   410 
   411 	map = malloc(nelem * sizeof (uint32_t));
   412 	for (i = 0; i < nelem; i++)
   413 		map[i] = i;
   414 
   415 	__qsort_with_data(base, nelem, map, &ctx);
   416 
   417 	return map;
   418 }
   419 
   420 static int
   421 versioncmp(const char *s1, const char *s2)
   422 {
   423 	const char *p1, *p2;
   424 	long n1, n2;
   425 	int res;
   426 
   427 	n1 = strtol(s1, (char **) &p1, 0);
   428 	n2 = strtol(s2, (char **) &p2, 0);
   429 
   430 	/* Epoch; if one but not the other has an epoch set, default
   431 	 * the epoch-less version to 0. */
   432 	res = (*p1 == ':') - (*p2 == ':');
   433 	if (res < 0) {
   434 		n1 = 0;
   435 		p1 = s1;
   436 		p2++;
   437 	} else if (res > 0) {
   438 		p1++;
   439 		n2 = 0;
   440 		p2 = s2;
   441 	}
   442 
   443 	if (n1 != n2)
   444 		return n1 - n2;
   445 	while (*p1 && *p2) {
   446 		if (*p1 != *p2)
   447 			return *p1 - *p2;
   448 		p1++;
   449 		p2++;
   450 		if (isdigit(*p1) && isdigit(*p2))
   451 			return versioncmp(p1, p2);
   452 	}
   453 
   454 	return *p1 - *p2;
   455 }
   456 
   457 static int
   458 compare_packages(const void *p1, const void *p2, void *data)
   459 {
   460 	const struct razor_package *pkg1 = p1, *pkg2 = p2;
   461 	struct razor_set *set = data;
   462 	char *pool = set->string_pool.data;
   463 
   464 	if (pkg1->name == pkg2->name)
   465 		return versioncmp(&pool[pkg1->version], &pool[pkg2->version]);
   466 	else
   467 		return strcmp(&pool[pkg1->name], &pool[pkg2->name]);
   468 }
   469 
   470 static int
   471 compare_properties(const void *p1, const void *p2, void *data)
   472 {
   473 	const struct razor_property *prop1 = p1, *prop2 = p2;
   474 	struct razor_set *set = data;
   475 	char *pool = set->string_pool.data;
   476 
   477 	if (prop1->name == prop2->name) {
   478 		if (prop1->relation == prop2->relation)
   479 			return versioncmp(&pool[prop1->version],
   480 					  &pool[prop2->version]);
   481 		else
   482 			return prop1->relation - prop2->relation;
   483 	} else if ((prop1->name & RAZOR_ENTRY_MASK) == (prop2->name & RAZOR_ENTRY_MASK))
   484 		return (prop1->name >> 30) - (prop2->name >> 30);
   485 	else
   486 		return strcmp(&pool[prop1->name & RAZOR_ENTRY_MASK],
   487 			      &pool[prop2->name & RAZOR_ENTRY_MASK]);
   488 }
   489 
   490 static uint32_t *
   491 uniqueify_properties(struct razor_set *set)
   492 {
   493 	struct razor_property *rp, *up, *rp_end;
   494 	struct array *pkgs, *p;
   495 	uint32_t *map, *rmap, *r;
   496 	int i, count, unique;
   497 
   498 	count = set->properties.size / sizeof(struct razor_property);
   499 	map = qsort_with_data(set->properties.data,
   500 			      count,
   501 			      sizeof(struct razor_property),
   502 			      compare_properties,
   503 			      set);
   504 
   505 	rp_end = set->properties.data + set->properties.size;
   506 	rmap = malloc(count * sizeof *map);
   507 	pkgs = zalloc(count * sizeof *pkgs);
   508 	for (rp = set->properties.data, up = rp, i = 0; rp < rp_end; rp++, i++) {
   509 		if (rp->name != up->name || rp->relation != up->relation ||
   510 		    rp->version != up->version) {
   511 			up++;
   512 			up->name = rp->name;
   513 			up->relation = rp->relation;
   514 			up->version = rp->version;
   515 		}
   516 
   517 		unique = up - (struct razor_property *) set->properties.data;
   518 		rmap[map[i]] = unique;
   519 		r = array_add(&pkgs[unique], sizeof *r);
   520 		*r = rp->packages;
   521 	}
   522 	free(map);
   523 
   524 	up++;
   525 	set->properties.size = (void *) up - set->properties.data;
   526 	rp_end = up;
   527 	for (rp = set->properties.data, p = pkgs; rp < rp_end; rp++, p++) {
   528 		rp->packages =
   529 			add_to_property_pool(&set->package_pool, p);
   530 		array_release(p);
   531 	}
   532 
   533 	free(pkgs);
   534 
   535 	return rmap;
   536 }
   537 
   538 static void
   539 remap_links(struct array *links, uint32_t *map)
   540 {
   541 	uint32_t *p, *end;
   542 
   543 	end = links->data + links->size;
   544 	for (p = links->data; p < end; p++)
   545 		*p = map[*p & RAZOR_ENTRY_MASK] | (*p & ~RAZOR_ENTRY_MASK);
   546 }
   547 
   548 static int
   549 compare_filenames(const void *p1, const void *p2, void *data)
   550 {
   551 	const struct import_entry *e1 = p1;
   552 	const struct import_entry *e2 = p2;
   553 
   554 	return strcmp(e1->name, e2->name);
   555 }
   556 
   557 static void
   558 count_entries(struct import_directory *d)
   559 {
   560 	struct import_directory *p, *end;
   561 
   562 	p = d->files.data;
   563 	end = d->files.data + d->files.size;
   564 	d->count = 0;
   565 	while (p < end) {
   566 		count_entries(p);
   567 		d->count += p->count + 1;
   568 		p++;
   569 	}		
   570 }
   571 
   572 static void
   573 serialize_files(struct razor_set *set,
   574 		struct import_directory *d, struct array *array)
   575 {
   576 	struct import_directory *p, *end;
   577 	struct razor_entry *e = NULL;
   578 	uint32_t s;
   579 
   580 	p = d->files.data;
   581 	end = d->files.data + d->files.size;
   582 	s = array->size / sizeof *e + d->files.size / sizeof *p;
   583 	while (p < end) {
   584 		e = array_add(array, sizeof *e);
   585 		e->name = p->name;
   586 		e->start = p->count > 0 ? s : 0;
   587 		s += p->count;
   588 
   589 		e->packages = add_to_property_pool(&set->package_pool,
   590 						   &p->packages);
   591 		array_release(&p->packages);
   592 		p++;
   593 	}		
   594 	if (e != NULL)
   595 		e->name |= RAZOR_ENTRY_LAST;
   596 
   597 	p = d->files.data;
   598 	end = d->files.data + d->files.size;
   599 	while (p < end) {
   600 		serialize_files(set, p, array);
   601 		p++;
   602 	}
   603 }
   604 
   605 static void
   606 remap_property_package_links(struct array *properties, uint32_t *rmap)
   607 {
   608 	struct razor_property *p, *end;
   609 
   610 	end = properties->data + properties->size;
   611 	for (p = properties->data; p < end; p++)
   612 		if (p->packages & RAZOR_IMMEDIATE)
   613 			p->packages = rmap[p->packages & RAZOR_ENTRY_MASK] |
   614 				RAZOR_IMMEDIATE;
   615 }
   616 
   617 static void
   618 build_file_tree(struct razor_importer *importer)
   619 {
   620 	int count, i, length;
   621 	struct import_entry *filenames;
   622 	char *f, *end;
   623 	uint32_t name, *r;
   624 	char dirname[256];
   625 	struct import_directory *d, root;
   626 	struct razor_entry *e;
   627 
   628 	count = importer->files.size / sizeof (struct import_entry);
   629 	qsort_with_data(importer->files.data,
   630 			count,
   631 			sizeof (struct import_entry),
   632 			compare_filenames,
   633 			NULL);
   634 
   635 	root.name = hashtable_tokenize(&importer->table, "");
   636 	array_init(&root.files);
   637 	array_init(&root.packages);
   638 	root.last = NULL;
   639 
   640 	filenames = importer->files.data;
   641 	for (i = 0; i < count; i++) {
   642 		f = filenames[i].name;
   643 		if (*f != '/')
   644 			continue;
   645 		f++;
   646 
   647 		d = &root;
   648 		while (*f) {
   649 			end = strchr(f, '/');
   650 			if (end == NULL)
   651 				end = f + strlen(f);
   652 			length = end - f;
   653 			memcpy(dirname, f, length);
   654 			dirname[length] ='\0';
   655 			name = hashtable_tokenize(&importer->table, dirname);
   656 			if (d->last == NULL || d->last->name != name) {
   657 				d->last = array_add(&d->files, sizeof *d);
   658 				d->last->name = name;
   659 				d->last->last = NULL;
   660 				array_init(&d->last->files);
   661 				array_init(&d->last->packages);
   662 			}
   663 			d = d->last;				
   664 			f = end + 1;
   665 			if (*end == '\0')
   666 				break;
   667 		}
   668 
   669 		r = array_add(&d->packages, sizeof *r);
   670 		*r = filenames[i].package;
   671 		free(filenames[i].name);
   672 	}
   673 
   674 	count_entries(&root);
   675 	array_init(&importer->set->files);
   676 
   677 	e = array_add(&importer->set->files, sizeof *e);
   678 	e->name = root.name | RAZOR_ENTRY_LAST;
   679 	e->start = 1;
   680 	e->packages = ~0;
   681 
   682 	serialize_files(importer->set, &root, &importer->set->files);
   683 
   684 	array_release(&importer->files);
   685 }
   686 
   687 static void
   688 build_package_file_lists(struct razor_set *set, uint32_t *rmap)
   689 {
   690 	struct razor_package *p, *packages;
   691 	struct array *pkgs;
   692 	struct razor_entry *e, *end;
   693 	uint32_t *r, *q;
   694 	int i, count;
   695 
   696 	count = set->packages.size / sizeof *p;
   697 	pkgs = zalloc(count * sizeof *pkgs);
   698 
   699 	end = set->files.data + set->files.size;
   700 	for (e = set->files.data; e < end; e++) {
   701 		if (e->packages == ~0) {
   702 			continue;
   703 		} else if (e->packages & RAZOR_IMMEDIATE) {
   704 			e->packages = rmap[e->packages & RAZOR_ENTRY_MASK] |
   705 				RAZOR_IMMEDIATE;
   706 			r = &e->packages;
   707 		} else {
   708 			r = (uint32_t *) set->package_pool.data + e->packages;
   709 		}
   710 
   711 		while (1) {
   712 			q = array_add(&pkgs[*r & RAZOR_ENTRY_MASK], sizeof *q);
   713 			*q = e - (struct razor_entry *) set->files.data;
   714 			if (*r++ & RAZOR_IMMEDIATE)
   715 				break;
   716 		}
   717 	}
   718 
   719 	packages = set->packages.data;
   720 	for (i = 0; i < count; i++) {
   721 		packages[i].files =
   722 			add_to_property_pool(&set->file_pool, &pkgs[i]);
   723 		array_release(&pkgs[i]);
   724 	}
   725 	free(pkgs);
   726 }
   727 
   728 struct razor_set *
   729 razor_importer_finish(struct razor_importer *importer)
   730 {
   731 	struct razor_set *set;
   732 	uint32_t *map, *rmap;
   733 	int i, count;
   734 
   735 	map = uniqueify_properties(importer->set);
   736 	remap_links(&importer->set->property_pool, map);
   737 	free(map);
   738 
   739 	count = importer->set->packages.size / sizeof(struct razor_package);
   740 	map = qsort_with_data(importer->set->packages.data,
   741 			      count,
   742 			      sizeof(struct razor_package),
   743 			      compare_packages,
   744 			      importer->set);
   745 
   746 	rmap = malloc(count * sizeof *rmap);
   747 	for (i = 0; i < count; i++)
   748 		rmap[map[i]] = i;
   749 	free(map);
   750 
   751 	build_file_tree(importer);
   752 	remap_links(&importer->set->package_pool, rmap);
   753 	build_package_file_lists(importer->set, rmap);
   754 	remap_property_package_links(&importer->set->properties, rmap);
   755 	free(rmap);
   756 
   757 	set = importer->set;
   758 	hashtable_release(&importer->table);
   759 	free(importer);
   760 
   761 	return set;
   762 }
   763 
   764 struct razor_package_iterator {
   765 	struct razor_set *set;
   766 	struct razor_package *package, *end;
   767 	uint32_t *index;
   768 	int last;
   769 };
   770 
   771 struct razor_package_iterator *
   772 razor_package_iterator_create_with_index(struct razor_set *set,
   773 					 uint32_t *index)
   774 {
   775 	struct razor_package_iterator *pi;
   776 
   777 	pi = zalloc(sizeof *pi);
   778 	pi->set = set;
   779 	pi->end = set->packages.data + set->packages.size;
   780 	pi->package = set->packages.data;
   781 	pi->index = index;
   782 
   783 	return pi;
   784 }
   785 
   786 struct razor_package_iterator *
   787 razor_package_iterator_create(struct razor_set *set)
   788 {
   789 	return razor_package_iterator_create_with_index(set, NULL);
   790 }
   791 
   792 struct razor_package_iterator *
   793 razor_package_iterator_create_for_property(struct razor_set *set,
   794 					   struct razor_property *property)
   795 {
   796 	uint32_t *index;
   797 
   798 	if (property->packages & RAZOR_IMMEDIATE)
   799 		index = &property->packages;
   800 	else
   801 		index = (uint32_t *)
   802 			set->package_pool.data + property->packages;
   803 
   804 	return razor_package_iterator_create_with_index(set, index);
   805 }
   806 
   807 int
   808 razor_package_iterator_next(struct razor_package_iterator *pi,
   809 			    struct razor_package **package,
   810 			    const char **name, const char **version)
   811 {
   812 	char *pool;
   813 	int valid;
   814 	struct razor_package *p, *packages;
   815 
   816 	if (pi->index) {
   817 		packages = pi->set->packages.data;
   818 		p = &packages[*pi->index & RAZOR_ENTRY_MASK];
   819 		valid = !pi->last;
   820 		pi->last = (*pi->index++ & RAZOR_IMMEDIATE) != 0;
   821 	} else {
   822 		p = pi->package++;
   823 		valid = p < pi->end;
   824 	}			
   825 
   826 	if (valid) {
   827 		pool = pi->set->string_pool.data;
   828 		*package = p;
   829 		*name = &pool[p->name & RAZOR_ENTRY_MASK];
   830 		*version = &pool[p->version];
   831 	} else {
   832 		*package = NULL;
   833 	}
   834 
   835 	return valid;
   836 }
   837 
   838 void
   839 razor_package_iterator_destroy(struct razor_package_iterator *pi)
   840 {
   841 	free(pi);
   842 }
   843 
   844 struct razor_package *
   845 razor_set_get_package(struct razor_set *set, const char *package)
   846 {
   847 	struct razor_package_iterator *pi;
   848 	struct razor_package *p;
   849 	const char *name, *version;
   850 
   851 	pi = razor_package_iterator_create(set);
   852 	while (razor_package_iterator_next(pi, &p, &name, &version)) {
   853 		if (strcmp(package, name) == 0)
   854 			break;
   855 	}
   856 	razor_package_iterator_destroy(pi);
   857 
   858 	return p;
   859 }
   860 
   861 struct razor_property_iterator {
   862 	struct razor_set *set;
   863 	struct razor_property *property, *end;
   864 	uint32_t *index;
   865 	int last;
   866 };
   867 
   868 struct razor_property_iterator *
   869 razor_property_iterator_create(struct razor_set *set,
   870 			       struct razor_package *package)
   871 {
   872 	struct razor_property_iterator *pi;
   873 
   874 	pi = zalloc(sizeof *pi);
   875 	pi->set = set;
   876 	pi->end = set->properties.data + set->properties.size;
   877 	pi->property = set->properties.data;
   878 
   879 	if (package)
   880 		pi->index = (uint32_t *)
   881 			set->property_pool.data + package->properties;
   882 
   883 	return pi;
   884 }
   885 
   886 int
   887 razor_property_iterator_next(struct razor_property_iterator *pi,
   888 			     struct razor_property **property,
   889 			     const char **name,
   890 			     enum razor_version_relation *relation,
   891 			     const char **version,
   892 			     enum razor_property_type *type)
   893 {
   894 	char *pool;
   895 	int valid;
   896 	struct razor_property *p, *properties;
   897 
   898 	if (pi->index) {
   899 		properties = pi->set->properties.data;
   900 		p = &properties[*pi->index & RAZOR_ENTRY_MASK];
   901 		valid = !pi->last;
   902 		pi->last = (*pi->index++ & RAZOR_IMMEDIATE) != 0;
   903 	} else {
   904 		p = pi->property++;
   905 		valid = p < pi->end;
   906 	}			
   907 
   908 	if (valid) {
   909 		pool = pi->set->string_pool.data;
   910 		*property = p;
   911 		*name = &pool[p->name & RAZOR_ENTRY_MASK];
   912 		*relation = p->relation;
   913 		*version = &pool[p->version];
   914 		*type = p->name >> 30;
   915 	} else {
   916 		*property = NULL;
   917 	}
   918 
   919 	return valid;
   920 }
   921 
   922 void
   923 razor_property_iterator_destroy(struct razor_property_iterator *pi)
   924 {
   925 	free(pi);
   926 }
   927 
   928 static struct razor_entry *
   929 find_entry(struct razor_set *set, struct razor_entry *dir, const char *pattern)
   930 {
   931 	struct razor_entry *e;
   932 	const char *n, *pool = set->string_pool.data;
   933 	int len;
   934 
   935 	e = (struct razor_entry *) set->files.data + dir->start;
   936 	do {
   937 		n = pool + (e->name & RAZOR_ENTRY_MASK);
   938 		if (strcmp(pattern + 1, n) == 0)
   939 			return e;
   940 		len = strlen(n);
   941 		if (e->start != 0 && strncmp(pattern + 1, n, len) == 0 &&
   942 		    pattern[len + 1] == '/') {
   943 			return find_entry(set, e, pattern + len + 1);
   944 		}
   945 	} while (((e++)->name & RAZOR_ENTRY_LAST) == 0);
   946 
   947 	return NULL;
   948 }
   949 
   950 static void
   951 list_dir(struct razor_set *set, struct razor_entry *dir,
   952 	 const char *prefix, const char *pattern)
   953 {
   954 	struct razor_entry *e;
   955 	const char *n, *pool = set->string_pool.data;
   956 
   957 	e = (struct razor_entry *) set->files.data + dir->start;
   958 	do {
   959 		n = pool + (e->name & RAZOR_ENTRY_MASK);
   960 		if (pattern && pattern[0] && fnmatch(pattern, n, 0) != 0)
   961 			continue;
   962 		printf("%s/%s%s\n", prefix, n, e->start > 0 ? "/" : "");
   963 	} while (((e++)->name & RAZOR_ENTRY_LAST) == 0);
   964 }
   965 
   966 void
   967 razor_set_list_files(struct razor_set *set, const char *pattern)
   968 {
   969 	struct razor_entry *e;
   970 	char buffer[512], *p, *base;
   971 
   972 	if (pattern == NULL)
   973 		pattern = "/";
   974 
   975 	strcpy(buffer, pattern);
   976 	e = find_entry(set, set->files.data, buffer);
   977 	if (e && e->start > 0) {
   978 		base = NULL;
   979 	} else {
   980 		p = strrchr(buffer, '/');
   981 		if (p) {
   982 			*p = '\0';
   983 			base = p + 1;
   984 		} else {
   985 			base = NULL;
   986 		}
   987 	}
   988 	e = find_entry(set, set->files.data, buffer);
   989 	if (e->start != 0)
   990 		list_dir(set, e, buffer, base);
   991 }
   992 
   993 struct razor_package_iterator *
   994 razor_package_iterator_create_for_file(struct razor_set *set,
   995 				       const char *filename)
   996 {
   997 	struct razor_entry *entry;
   998 	uint32_t *index;
   999 
  1000 	entry = find_entry(set, set->files.data, filename);
  1001 	if (entry == NULL)
  1002 		return NULL;
  1003 	
  1004 	if (entry->packages & RAZOR_IMMEDIATE)
  1005 		index = &entry->packages;
  1006 	else
  1007 		index = (uint32_t *)
  1008 			set->package_pool.data + entry->packages;
  1009 
  1010 	return razor_package_iterator_create_with_index(set, index);
  1011 }
  1012 
  1013 static uint32_t *
  1014 list_package_files(struct razor_set *set, uint32_t *r,
  1015 		   struct razor_entry *dir, uint32_t end,
  1016 		   char *prefix)
  1017 {
  1018 	struct razor_entry *e, *f, *entries;
  1019 	uint32_t next, file;
  1020 	char *pool;
  1021 	int len;
  1022 	
  1023 	entries = (struct razor_entry *) set->files.data;
  1024 	pool = set->string_pool.data;
  1025 
  1026 	e = entries + dir->start;
  1027 	do {
  1028 		if (entries + (*r & RAZOR_ENTRY_MASK) == e) {
  1029 			printf("%s/%s\n", prefix,
  1030 			       pool + (e->name & RAZOR_ENTRY_MASK));
  1031 			if (*r & RAZOR_ENTRY_LAST)
  1032 				return NULL;
  1033 			r++;
  1034 			if ((*r & RAZOR_ENTRY_MASK) >= end)
  1035 				return r;
  1036 		}
  1037 	} while (!((e++)->name & RAZOR_ENTRY_LAST));
  1038 
  1039 	e = entries + dir->start;
  1040 	do {
  1041 		if (e->start == 0)
  1042 			continue;
  1043 
  1044 		if (e->name & RAZOR_ENTRY_LAST)
  1045 			next = end;
  1046 		else {
  1047 			f = e + 1; 
  1048 			while (f->start == 0 && !(f->name & RAZOR_ENTRY_LAST))
  1049 				f++;
  1050 			if (f->start == 0)
  1051 				next = end;
  1052 			else
  1053 				next = f->start;
  1054 		}
  1055 
  1056 		file = *r & RAZOR_ENTRY_MASK;
  1057 		if (e->start <= file && file < next) {
  1058 			len = strlen(prefix);
  1059 			prefix[len] = '/';
  1060 			strcpy(prefix + len + 1,
  1061 			       pool + (e->name & RAZOR_ENTRY_MASK));
  1062 			r = list_package_files(set, r, e, next, prefix);
  1063 			prefix[len] = '\0';
  1064 		}
  1065 	} while (!((e++)->name & RAZOR_ENTRY_LAST) && r != NULL);
  1066 
  1067 	return r;
  1068 }
  1069 
  1070 void
  1071 razor_set_list_package_files(struct razor_set *set, const char *name)
  1072 {
  1073 	struct razor_package *package;
  1074 	uint32_t *r, end;
  1075 	char buffer[512];
  1076 
  1077 	package = razor_set_get_package(set, name);
  1078 
  1079 	r = (uint32_t *) set->file_pool.data + package->files;
  1080 	end = set->files.size / sizeof (struct razor_entry);
  1081 	buffer[0] = '\0';
  1082 	list_package_files(set, r, set->files.data, end, buffer);
  1083 }
  1084 
  1085 static void
  1086 razor_set_validate(struct razor_set *set, struct array *unsatisfied)
  1087 {
  1088 	struct razor_property *r, *p, *end;
  1089 	uint32_t *u;
  1090 	char *pool;
  1091 
  1092 	end = set->properties.data + set->properties.size;
  1093 	pool = set->string_pool.data;
  1094 	
  1095 	for (r = set->properties.data, p = r; r < end; r++) {
  1096 		if (r->name >> 30 != RAZOR_PROPERTY_REQUIRES)
  1097 			continue;
  1098 
  1099 		if ((r->name & RAZOR_ENTRY_MASK) != (p->name & RAZOR_ENTRY_MASK)) {
  1100 			p = r;
  1101 			while (p < end && p->name == r->name)
  1102 				p++;
  1103 		}
  1104 
  1105 		/* If there is more than one version of a provides,
  1106 		 * seek to the end for the highest version. */
  1107 		/* FIXME: This doesn't work if we have a series of
  1108 		 * requires a = 1, provides a = 1, requires a = 2,
  1109 		 * provides a = 2, as the kernel and kernel-devel
  1110 		 * does.*/
  1111 		while (p + 1 < end && p->name == (p + 1)->name)
  1112 			p++;
  1113 
  1114 		/* FIXME: We need to track property flags (<, <=, =
  1115 		 * etc) to properly determine if a requires is
  1116 		 * satisfied.  The current code doesn't track that the
  1117 		 * requires a = 1 isn't satisfied by a = 2 provides. */
  1118 
  1119 		if (p == end ||
  1120 		    (p->name >> 30) != RAZOR_PROPERTY_PROVIDES ||
  1121 		    (r->name & RAZOR_ENTRY_MASK) != (p->name & RAZOR_ENTRY_MASK) ||
  1122 		    versioncmp(&pool[r->version], &pool[p->version]) > 0) {
  1123 			/* FIXME: We ignore file requires for now. */
  1124 			if (pool[r->name & RAZOR_ENTRY_MASK] == '/')
  1125 				continue;
  1126 			u = array_add(unsatisfied, sizeof *u);
  1127 			*u = r - (struct razor_property *) set->properties.data;
  1128 		}
  1129 	}
  1130 }
  1131 
  1132 void
  1133 razor_set_list_unsatisfied(struct razor_set *set)
  1134 {
  1135 	struct array unsatisfied;
  1136 	struct razor_property *properties, *r;
  1137 	uint32_t *u, *end;
  1138 	char *pool;
  1139 
  1140 	array_init(&unsatisfied);
  1141 	razor_set_validate(set, &unsatisfied);
  1142 
  1143 	end = unsatisfied.data + unsatisfied.size;
  1144 	properties = set->properties.data;
  1145 	pool = set->string_pool.data;
  1146 
  1147 	for (u = unsatisfied.data; u < end; u++) {
  1148 		r = properties + *u;
  1149 		if (pool[r->version] == '\0')
  1150 			printf("%ss not satisfied\n",
  1151 			       &pool[r->name & RAZOR_ENTRY_MASK]);
  1152 		else
  1153 			printf("%s-%s not satisfied\n",
  1154 			       &pool[r->name & RAZOR_ENTRY_MASK],
  1155 			       &pool[r->version]);
  1156 	}
  1157 
  1158 	array_release(&unsatisfied);
  1159 }
  1160 
  1161 #define UPSTREAM_SOURCE 0x80000000ul
  1162 #define INDEX_MASK 0x00fffffful
  1163 
  1164 struct source {
  1165 	struct razor_set *set;
  1166 	uint32_t *property_map;
  1167 };
  1168 
  1169 struct razor_merger {
  1170 	struct razor_set *set;
  1171 	struct hashtable table;
  1172 	struct source source1;
  1173 	struct source source2;
  1174 };
  1175 
  1176 static struct razor_merger *
  1177 razor_merger_create(struct razor_set *set1, struct razor_set *set2)
  1178 {
  1179 	struct razor_merger *merger;
  1180 	int count;
  1181 	size_t size;
  1182 
  1183 	merger = zalloc(sizeof *merger);
  1184 	merger->set = razor_set_create();
  1185 	hashtable_init(&merger->table, &merger->set->string_pool);
  1186 
  1187 	count = set1->properties.size / sizeof (struct razor_property);
  1188 	size = count * sizeof merger->source1.property_map[0];
  1189 	merger->source1.property_map = zalloc(size);
  1190 	merger->source1.set = set1;
  1191 
  1192 	count = set2->properties.size / sizeof (struct razor_property);
  1193 	size = count * sizeof merger->source2.property_map[0];
  1194 	merger->source2.property_map = zalloc(size);
  1195 	merger->source2.set = set2;
  1196 
  1197 	return merger;
  1198 }
  1199 
  1200 static void
  1201 add_package(struct razor_merger *merger,
  1202 	    struct razor_package *package, struct source *source,
  1203 	    uint32_t flags)
  1204 {
  1205 	char *pool;
  1206 	uint32_t *r;
  1207 	struct razor_package *p;
  1208 
  1209 	pool = source->set->string_pool.data;
  1210 	p = array_add(&merger->set->packages, sizeof *p);
  1211 	p->name = hashtable_tokenize(&merger->table, &pool[package->name]);
  1212 	p->name |= flags;
  1213 	p->version = hashtable_tokenize(&merger->table,
  1214 					&pool[package->version]);
  1215 	p->properties = package->properties;
  1216 
  1217 	if (package->properties & RAZOR_IMMEDIATE)
  1218 		r = &package->properties;
  1219 	else
  1220 		r = (uint32_t *)
  1221 			source->set->property_pool.data + package->properties;
  1222 	while (1) {
  1223 		source->property_map[*r & RAZOR_ENTRY_MASK] = 1;
  1224 		if (*r++ & RAZOR_IMMEDIATE)
  1225 			break;
  1226 	}
  1227 }
  1228 
  1229 
  1230 /* Build the new package list sorted by merging the two package lists.
  1231  * Build new string pool as we go. */
  1232 static void
  1233 merge_packages(struct razor_merger *merger, struct array *packages)
  1234 {
  1235 	struct razor_package *upstream_packages, *p, *s, *send;
  1236 	struct source *source1, *source2;
  1237 	char *spool, *upool;
  1238 	uint32_t *u, *uend;
  1239 	int cmp;
  1240 
  1241 	source1 = &merger->source1;
  1242 	source2 = &merger->source2;
  1243 	upstream_packages = source2->set->packages.data;
  1244 
  1245 	u = packages->data;
  1246 	uend = packages->data + packages->size;
  1247 	upool = source2->set->string_pool.data;
  1248 
  1249 	s = source1->set->packages.data;
  1250 	send = source1->set->packages.data + source1->set->packages.size;
  1251 	spool = source1->set->string_pool.data;
  1252 
  1253 	while (s < send) {
  1254 		p = upstream_packages + *u;
  1255 
  1256 		if (u < uend)
  1257 			cmp = strcmp(&spool[s->name], &upool[p->name]);
  1258 		if (u >= uend || cmp < 0) {
  1259 			add_package(merger, s, source1, 0);
  1260 			s++;
  1261 		} else if (cmp == 0) {
  1262 			add_package(merger, p, source2, UPSTREAM_SOURCE);
  1263 			s++;
  1264 			u++;
  1265 		} else {
  1266 			add_package(merger, p, source2, UPSTREAM_SOURCE);
  1267 			u++;
  1268 		}
  1269 	}
  1270 }
  1271 
  1272 static uint32_t
  1273 add_property(struct razor_merger *merger,
  1274 	     const char *name, enum razor_version_relation relation,
  1275 	     const char *version, int type)
  1276 {
  1277 	struct razor_property *p;
  1278 
  1279 	p = array_add(&merger->set->properties, sizeof *p);
  1280 	p->name = hashtable_tokenize(&merger->table, name) | (type << 30);
  1281 	p->relation = relation;
  1282 	p->version = hashtable_tokenize(&merger->table, version);
  1283 
  1284 	return p - (struct razor_property *) merger->set->properties.data;
  1285 }
  1286 
  1287 static void
  1288 merge_properties(struct razor_merger *merger)
  1289 {
  1290 	struct razor_property *p1, *p2;
  1291 	struct razor_set *set1, *set2;
  1292 	uint32_t *map1, *map2;
  1293 	int i, j, cmp, count1, count2;
  1294 	char *pool1, *pool2;
  1295 
  1296 	set1 = merger->source1.set;
  1297 	set2 = merger->source2.set;
  1298 	map1 = merger->source1.property_map;
  1299 	map2 = merger->source2.property_map;
  1300 
  1301 	i = 0;
  1302 	j = 0;
  1303 	pool1 = set1->string_pool.data;
  1304 	pool2 = set2->string_pool.data;
  1305 
  1306 	count1 = set1->properties.size / sizeof *p1;
  1307 	count2 = set2->properties.size / sizeof *p2;
  1308 	while (i < count1 || j < count2) {
  1309 		if (i < count1 && map1[i] == 0) {
  1310 			i++;
  1311 			continue;
  1312 		}
  1313 		if (j < count2 && map2[j] == 0) {
  1314 			j++;
  1315 			continue;
  1316 		}
  1317 		p1 = (struct razor_property *) set1->properties.data + i;
  1318 		p2 = (struct razor_property *) set2->properties.data + j;
  1319 		if (i < count1 && j < count2)
  1320 			cmp = strcmp(&pool1[p1->name & RAZOR_ENTRY_MASK],
  1321 				     &pool2[p2->name & RAZOR_ENTRY_MASK]);
  1322 		else if (i < count1)
  1323 			cmp = -1;
  1324 		else
  1325 			cmp = 1;
  1326 		if (cmp == 0)
  1327 			cmp = p1->relation - p2->relation;
  1328 		if (cmp == 0)
  1329 			cmp = versioncmp(&pool1[p1->version],
  1330 					 &pool2[p2->version]);
  1331 		if (cmp < 0) {
  1332 			map1[i++] = add_property(merger,
  1333 						 &pool1[p1->name & RAZOR_ENTRY_MASK],
  1334 						 p1->relation,
  1335 						 &pool1[p1->version],
  1336 						 (p1->name >> 30));
  1337 		} else if (cmp > 0) {
  1338 			map2[j++] = add_property(merger,
  1339 						 &pool2[p2->name & RAZOR_ENTRY_MASK],
  1340 						 p2->relation,
  1341 						 &pool2[p2->version],
  1342 						 (p2->name >> 30));
  1343 		} else  {
  1344 			map1[i++] = map2[j++] = add_property(merger,
  1345 							     &pool1[p1->name & RAZOR_ENTRY_MASK],
  1346 							     p1->relation,
  1347 							     &pool1[p1->version],
  1348 							     (p1->name >> 30));
  1349 		}
  1350 	}
  1351 }
  1352 
  1353 static uint32_t
  1354 emit_properties(struct array *source_pool, uint32_t index,
  1355 		uint32_t *map, struct array *pool)
  1356 {
  1357 	uint32_t r, *p, *q;
  1358 
  1359 	r = pool->size / sizeof *q;
  1360 	p = (uint32_t *) source_pool->data + index;
  1361 	while (1) {
  1362 		q = array_add(pool, sizeof *q);
  1363 		*q = map[*p & RAZOR_ENTRY_MASK] | (*p & ~RAZOR_ENTRY_MASK);
  1364 		if (*p++ & RAZOR_ENTRY_LAST)
  1365 			break;
  1366 	}
  1367 
  1368 	return r;
  1369 }
  1370 	
  1371 /* Rebuild property->packages maps.  We can't just remap these, as a
  1372  * property may have lost or gained a number of packages.  Allocate an
  1373  * array per property and loop through the packages and add them to
  1374  * the arrays for their properties. */
  1375 static void
  1376 rebuild_package_lists(struct razor_set *set)
  1377 {
  1378 	struct array *pkgs, *a;
  1379 	struct razor_package *pkg, *pkg_end;
  1380 	struct razor_property *prop, *prop_end;
  1381 	uint32_t *r, *q, *pool;
  1382 	int count;
  1383 
  1384 	count = set->properties.size / sizeof (struct razor_property);
  1385 	pkgs = zalloc(count * sizeof *pkgs);
  1386 	pkg_end = set->packages.data + set->packages.size;
  1387 	pool = set->property_pool.data;
  1388 
  1389 	for (pkg = set->packages.data; pkg < pkg_end; pkg++) {
  1390 		for (r = &pool[pkg->properties]; ; r++) {
  1391 			q = array_add(&pkgs[*r & RAZOR_ENTRY_MASK], sizeof *q);
  1392 			*q = pkg - (struct razor_package *) set->packages.data;
  1393 			if (*r & RAZOR_IMMEDIATE)
  1394 				break;
  1395 		}
  1396 	}
  1397 
  1398 	prop_end = set->properties.data + set->properties.size;
  1399 	a = pkgs;
  1400 	for (prop = set->properties.data; prop < prop_end; prop++, a++) {
  1401 		prop->packages =
  1402 			add_to_property_pool(&set->property_pool, a);
  1403 		array_release(a);
  1404 	}
  1405 	free(pkgs);
  1406 }
  1407 
  1408 struct razor_set *
  1409 razor_merger_finish(struct razor_merger *merger)
  1410 {
  1411 	struct razor_set *result;
  1412 
  1413 	result = merger->set;
  1414 	hashtable_release(&merger->table);
  1415 	free(merger);
  1416 
  1417 	return result;
  1418 }
  1419 
  1420 /* Add packages from 'upstream' to 'set'.  The packages to add are
  1421  * specified by the 'packages' array, which is a sorted list of
  1422  * package indexes.  Returns a newly allocated package set.  Does not
  1423  * enforce validity of the resulting package set.
  1424  *
  1425  * This looks more complicated than it is.  An easy way to merge two
  1426  * package sets would be to just use a razor_importer, but that
  1427  * requires resorting, and is thus O(n log n).  We can do this in a
  1428  * linear sweep, but it gets a little more complicated.
  1429  */
  1430 struct razor_set *
  1431 razor_set_add(struct razor_set *set, struct razor_set *upstream,
  1432 	      struct array *packages)
  1433 {
  1434 	struct razor_merger *merger;
  1435 	struct razor_package *p, *pend;
  1436 
  1437 	merger = razor_merger_create(set, upstream);
  1438 
  1439 	merge_packages(merger, packages);
  1440 
  1441 	/* As we built the package list, we filled out a bitvector of
  1442 	 * the properties that are referenced by the packages in the
  1443 	 * new set.  Now we do a parallel loop through the properties
  1444 	 * and emit those marked in the bit vector to the new set.  In
  1445 	 * the process, we update the bit vector to actually map from
  1446 	 * indices in the old property list to indices in the new
  1447 	 * property list for both sets. */
  1448 
  1449 	merge_properties(merger);
  1450 
  1451 	/* Now we loop through the packages again and emit the
  1452 	 * property lists, remapped to point to the new properties. */
  1453 
  1454 	pend = merger->set->packages.data + merger->set->packages.size;
  1455 	for (p = merger->set->packages.data; p < pend; p++) {
  1456 		struct source *src;
  1457 
  1458 		if (p->name & UPSTREAM_SOURCE)
  1459 			src = &merger->source2;
  1460 		else
  1461 			src = &merger->source1;
  1462 
  1463 		p->properties = emit_properties(&src->set->property_pool,
  1464 						p->properties,
  1465 						src->property_map,
  1466 						&merger->set->property_pool);
  1467 		p->name &= INDEX_MASK;
  1468 	}
  1469 
  1470 	rebuild_package_lists(merger->set);
  1471 
  1472 	return razor_merger_finish(merger);
  1473 }
  1474 
  1475 void
  1476 razor_set_satisfy(struct razor_set *set, struct array *unsatisfied,
  1477 		  struct razor_set *upstream, struct array *list)
  1478 {
  1479 	struct razor_property *requires, *r;
  1480 	struct razor_property *p, *pend;
  1481 	uint32_t *u, *end, *pkg, *package_pool;
  1482 	char *pool, *upool;
  1483 
  1484 	end = unsatisfied->data + unsatisfied->size;
  1485 	requires = set->properties.data;
  1486 	pool = set->string_pool.data;
  1487 
  1488 	p = upstream->properties.data;
  1489 	pend = upstream->properties.data + upstream->properties.size;
  1490 	upool = upstream->string_pool.data;
  1491 	package_pool = upstream->package_pool.data;
  1492 
  1493 	for (u = unsatisfied->data; u < end; u++) {
  1494 		r = requires + *u;
  1495 
  1496 		while (p < pend &&
  1497 		       strcmp(&pool[r->name & RAZOR_ENTRY_MASK],
  1498 			      &upool[p->name & RAZOR_ENTRY_MASK]) > 0 &&
  1499 		       (p->name >> 30) != RAZOR_PROPERTY_PROVIDES)
  1500 			p++;
  1501 		/* If there is more than one version of a provides,
  1502 		 * seek to the end for the highest version. */
  1503 		while (p + 1 < pend && p->name == (p + 1)->name)
  1504 			p++;
  1505 
  1506 		if (p == pend ||
  1507 		    strcmp(&pool[r->name & RAZOR_ENTRY_MASK],
  1508 			   &upool[p->name & RAZOR_ENTRY_MASK]) != 0 ||
  1509 		    versioncmp(&pool[r->version], &upool[p->version]) > 0) {
  1510 			/* Do we need to track unsatisfiable requires
  1511 			 * as we go, or should we just do a
  1512 			 * razor_set_validate() at the end? */
  1513 		} else {
  1514 			pkg = array_add(list, sizeof *pkg);
  1515 			/* We just pull in the first package that provides */
  1516 			if (p->packages & RAZOR_IMMEDIATE)
  1517 				*pkg = p->packages & RAZOR_ENTRY_MASK;
  1518 			else
  1519 				*pkg = package_pool[p->packages];
  1520 		}
  1521 	}	
  1522 }
  1523 
  1524 static void
  1525 find_packages(struct razor_set *set,
  1526 	      int count, const char **package_names, struct array *list)
  1527 {
  1528 	struct razor_package_iterator *pi;
  1529 	struct razor_package *p, *packages;
  1530 	const char *name, *version;
  1531 	uint32_t *r;
  1532 	int i;
  1533 
  1534 	packages = (struct razor_package *) set->packages.data;
  1535 	pi = razor_package_iterator_create(set);
  1536 
  1537 	while (razor_package_iterator_next(pi, &p, &name, &version)) {
  1538 		for (i = 0; i < count; i++) {
  1539 			if (strcmp(name, package_names[i]) == 0) {
  1540 				r = array_add(list, sizeof *r);
  1541 				*r = p - packages;
  1542 				break;
  1543 			}
  1544 		}
  1545 	}
  1546 
  1547 	razor_package_iterator_destroy(pi);
  1548 }
  1549 
  1550 static void
  1551 find_all_packages(struct razor_set *set,
  1552 		  struct razor_set *upstream, struct array *list)
  1553 {
  1554 	struct razor_package *p, *u, *pend, *uend;
  1555 	uint32_t *r;
  1556 	char *pool, *upool;
  1557 
  1558 	pend = set->packages.data + set->packages.size;
  1559 	pool = set->string_pool.data;
  1560 	u = upstream->packages.data;
  1561 	uend = upstream->packages.data + upstream->packages.size;
  1562 	upool = upstream->string_pool.data;
  1563 
  1564 	for (p = set->packages.data; p < pend; p++) {
  1565 		while (u < uend && strcmp(&pool[p->name], &upool[u->name]) > 0)
  1566 			u++;
  1567 		if (strcmp(&pool[p->name], &upool[u->name]) == 0) {
  1568 			r = array_add(list, sizeof *r);
  1569 			*r = u - (struct razor_package *) upstream->packages.data;
  1570 		}
  1571 	}
  1572 }
  1573 
  1574 struct razor_set *
  1575 razor_set_update(struct razor_set *set, struct razor_set *upstream,
  1576 		 int count, const char **packages)
  1577 {
  1578 	struct razor_set *new;
  1579 	struct razor_package *upackages;
  1580 	struct array list, unsatisfied;
  1581 	char *pool;
  1582 	uint32_t *u, *end;
  1583 	int total = 0;
  1584 
  1585 	array_init(&list);
  1586 	if (count > 0)
  1587 		find_packages(upstream, count, packages, &list);
  1588 	else
  1589 		find_all_packages(set, upstream, &list);
  1590 
  1591 	end = list.data + list.size;
  1592 	upackages = upstream->packages.data;
  1593 	pool = upstream->string_pool.data;
  1594 	total += list.size / sizeof *u;
  1595 
  1596 	while (list.size > 0) {
  1597 		new = razor_set_add(set, upstream, &list);
  1598 		array_release(&list);
  1599 		razor_set_destroy(set);
  1600 		set = new;
  1601 
  1602 		array_init(&unsatisfied);
  1603 		razor_set_validate(new, &unsatisfied);
  1604 		array_init(&list);
  1605 		razor_set_satisfy(new, &unsatisfied, upstream, &list);
  1606 		array_release(&unsatisfied);
  1607 
  1608 		end = list.data + list.size;
  1609 		upackages = upstream->packages.data;
  1610 		pool = upstream->string_pool.data;
  1611 		total += list.size / sizeof *u;
  1612 	}
  1613 
  1614 	array_release(&list);
  1615 
  1616 	return set;
  1617 }
  1618 
  1619 /* The diff order matters.  We should sort the packages so that a
  1620  * REMOVE of a package comes before the INSTALL, and so that all
  1621  * requires for a package have been installed before the package.
  1622  **/
  1623 
  1624 void
  1625 razor_set_diff(struct razor_set *set, struct razor_set *upstream,
  1626 	       razor_package_callback_t callback, void *data)
  1627 {
  1628 	struct razor_package_iterator *pi1, *pi2;
  1629 	struct razor_package *p1, *p2;
  1630 	const char *name1, *name2, *version1, *version2;
  1631 	int res;
  1632 
  1633 	pi1 = razor_package_iterator_create(set);
  1634 	pi2 = razor_package_iterator_create(upstream);
  1635 
  1636 	razor_package_iterator_next(pi1, &p1, &name1, &version1);
  1637 	razor_package_iterator_next(pi2, &p2, &name2, &version2);
  1638 
  1639 	while (p1 || p2) {
  1640 		if (p1 && p2) {
  1641 			res = strcmp(name1, name2);
  1642 			if (res == 0)
  1643 				res = versioncmp(version1, version2);
  1644 		} else {
  1645 			res = 0;
  1646 		}
  1647 
  1648 		if (p2 == NULL || res < 0)
  1649 			callback(name1, version1, NULL, data);
  1650 		else if (p1 == NULL || res > 0)
  1651 			callback(name2, NULL, version2, data);
  1652 
  1653 		if (p1 != NULL && res <= 0)
  1654 			razor_package_iterator_next(pi1, &p1,
  1655 						    &name1, &version1);
  1656 		if (p2 != NULL && res >= 0)
  1657 			razor_package_iterator_next(pi2, &p2,
  1658 						    &name2, &version2);
  1659 	}
  1660 
  1661 	razor_package_iterator_destroy(pi1);
  1662 	razor_package_iterator_destroy(pi2);
  1663 }