Mercurialrazor / file revision
summary | shortlog | changelog | graph | tags | branches | files | changeset | file | revisions | annotate | diff | raw
razor.c
author Kristian H?gsberg <krh@redhat.com>
Wed Sep 05 00:32:09 2007 -0400 (2007-09-05)
changeset 7 235f7daf817c
parent 6 4eeed5fbe6b7
child 8 7820b7d94662
permissions -rw-r--r--
Track provides in the package set. 
     1 #include <stdlib.h>

     2 #include <stdio.h>

     3 #include <string.h>

     4 #include <sys/types.h>

     5 #include <sys/stat.h>

     6 #include <sys/mman.h>

     7 #include <unistd.h>

     8 #include <fcntl.h>

     9 

    10 #include <expat.h>

    11 #include "sha1.h"

    12 

    13 struct array {

    14 	void *data;

    15 	int size, alloc;

    16 };

    17 

    18 static void *

    19 array_add(struct array *array, int size)

    20 {

    21 	int alloc;

    22 	void *data, *p;

    23 

    24 	if (array->alloc > 0)

    25 		alloc = array->alloc;

    26 	else

    27 		alloc = 1024;

    28 

    29 	while (alloc < array->size + size)

    30 		alloc *= 2;

    31 

    32 	if (array->alloc < alloc) {

    33 		data = realloc(array->data, alloc);

    34 		if (data == NULL)

    35 			return 0;

    36 		array->data = data;

    37 		array->alloc = alloc;

    38 	}

    39 

    40 	p = array->data + array->size;

    41 	array->size += size;

    42 

    43 	return p;

    44 }

    45 

    46 static int

    47 write_to_fd(int fd, void *p, size_t size)

    48 {

    49 	int rest, len;

    50 

    51 	rest = size;

    52 	while (rest > 0) {

    53 		len = write(fd, p, rest);

    54 		if (len < 0)

    55 			return -1;

    56 		rest -= len;

    57 	}

    58 

    59 	return 0;

    60 }

    61 

    62 static int

    63 write_to_file(const char *filename, void *p, size_t size)

    64 {

    65 	int fd, err;

    66 

    67 	fd = open(filename, O_CREAT | O_WRONLY | O_TRUNC, 0666);

    68 	if (fd < 0)

    69 		return -1;

    70 	err = write_to_fd(fd, p, size);

    71 	close(fd);

    72 

    73 	return err;

    74 }

    75 

    76 static void *

    77 zalloc(size_t size)

    78 {

    79 	void *p;

    80 

    81 	p = malloc(size);

    82 	memset(p, 0, size);

    83 

    84 	return p;

    85 }

    86 

    87 struct razor_set_header {

    88 	unsigned int magic;

    89 	unsigned int version;

    90 	struct { unsigned int type, offset; } sections[0];

    91 };

    92 

    93 #define RAZOR_MAGIC 0x7a7a7a7a

    94 #define RAZOR_VERSION 1

    95 #define RAZOR_BUCKETS 1

    96 #define RAZOR_STRINGS 2

    97 #define RAZOR_PACKAGES 3

    98 #define RAZOR_PROVIDES 4

    99 

   100 struct razor_package {

   101 	unsigned long name;

   102 	unsigned long version;

   103 };

   104 

   105 struct razor_provides {

   106 	unsigned long name;

   107 	unsigned long version;

   108 };

   109 

   110 struct razor_set {

   111 	struct array buckets;

   112 	struct array string_pool;

   113  	struct array packages;

   114  	struct array provides;

   115 	struct razor_set_header *header;

   116 };

   117 

   118 struct razor_set *

   119 razor_set_create(void)

   120 {

   121 	struct razor_set *set;

   122 	char *p;

   123 

   124 	set = zalloc(sizeof(struct razor_set));

   125 	p = array_add(&set->string_pool, 1);

   126 	*p = '\0';

   127 

   128 	return set;

   129 }

   130 

   131 struct razor_set *

   132 razor_set_open(const char *filename)

   133 {

   134 	struct razor_set *set;

   135 	struct stat stat;

   136 	unsigned int size, offset;

   137 	int fd, i;

   138 

   139 	set = zalloc(sizeof *set);

   140 	fd = open(filename, O_RDONLY);

   141 	if (fstat(fd, &stat) < 0)

   142 		return NULL;

   143 	set->header = mmap(NULL, stat.st_size, PROT_READ, MAP_PRIVATE, fd, 0);

   144 	if (set->header == MAP_FAILED) {

   145 		free(set);

   146 		return NULL;

   147 	}

   148 

   149 	for (i = 0; i < set->header->sections[i].type; i++) {

   150 		offset = set->header->sections[i].offset;

   151 		size = set->header->sections[i + 1].offset - offset;

   152 

   153 		switch (set->header->sections[i].type) {

   154 		case RAZOR_BUCKETS:

   155 			set->buckets.data = (void *) set->header + offset;

   156 			set->buckets.size = size;

   157 			set->buckets.alloc = size;

   158 			break;

   159 		case RAZOR_STRINGS:

   160 			set->string_pool.data = (void *) set->header + offset;

   161 			set->string_pool.size = size;

   162 			set->string_pool.alloc = size;

   163 			break;

   164 		case RAZOR_PACKAGES:

   165 			set->packages.data = (void *) set->header + offset;

   166 			set->packages.size = size;

   167 			set->packages.size = size;

   168 			break;

   169 		case RAZOR_PROVIDES:

   170 			set->provides.data = (void *) set->header + offset;

   171 			set->provides.size = size;

   172 			set->provides.size = size;

   173 			break;

   174 		}

   175 	}

   176 	close(fd);

   177 

   178 	return set;

   179 }

   180 

   181 void

   182 razor_set_destroy(struct razor_set *set)

   183 {

   184 	unsigned int size;

   185 	int i;

   186 

   187 	if (set->header) {

   188 		for (i = 0; set->header->sections[i].type; i++)

   189 			;

   190 		size = set->header->sections[i].type;

   191 		munmap(set->header, size);

   192 	} else {

   193 		free(set->buckets.data);

   194 		free(set->string_pool.data);

   195 		free(set->packages.data);

   196 	}

   197 

   198 	free(set);

   199 }

   200 

   201 static int

   202 razor_set_write(struct razor_set *set, const char *filename)

   203 {

   204 	char data[4096];

   205 	struct razor_set_header *header = (struct razor_set_header *) data;

   206 	int fd, pool_size, packages_size, provides_size;

   207 

   208 	/* Align these to pages sizes */

   209 	pool_size = (set->string_pool.size + 4095) & ~4095;

   210 	packages_size = (set->packages.size + 4095) & ~4095;

   211 	provides_size = (set->provides.size + 4095) & ~4095;

   212 

   213 	memset(data, 0, sizeof data);

   214 	header->magic = RAZOR_MAGIC;

   215 	header->version = RAZOR_VERSION;

   216 

   217 	header->sections[0].type = RAZOR_BUCKETS;

   218 	header->sections[0].offset = sizeof data;

   219 

   220 	header->sections[1].type = RAZOR_STRINGS;

   221 	header->sections[1].offset =

   222 		header->sections[0].offset + set->buckets.alloc;

   223 

   224 	header->sections[2].type = RAZOR_PACKAGES;

   225 	header->sections[2].offset =

   226 		header->sections[1].offset + pool_size;

   227 

   228 	header->sections[3].type = RAZOR_PROVIDES;

   229 	header->sections[3].offset =

   230 		header->sections[2].offset + packages_size;

   231 

   232 	header->sections[4].type = 0;

   233 	header->sections[4].offset =

   234 		header->sections[3].offset + provides_size;

   235 

   236 	fd = open(filename, O_CREAT | O_WRONLY | O_TRUNC, 0666);

   237 	if (fd < 0)

   238 		return -1;

   239 

   240 	write_to_fd(fd, data, sizeof data);

   241 	write_to_fd(fd, set->buckets.data, set->buckets.alloc);

   242 	write_to_fd(fd, set->string_pool.data, pool_size);

   243 	write_to_fd(fd, set->packages.data, packages_size);

   244 	write_to_fd(fd, set->provides.data, provides_size);

   245 

   246 	return 0;

   247 }

   248 

   249 static unsigned int

   250 hash_string(const char *key)

   251 {

   252 	const char *p;

   253 	unsigned int hash = 0;

   254 

   255 	for (p = key; *p; p++)

   256 		hash = (hash << 2) ^ *p;

   257 

   258 	return hash;

   259 }

   260 

   261 unsigned long

   262 razor_set_lookup(struct razor_set *set, const char *key)

   263 {

   264 	unsigned int mask, start, i;

   265 	unsigned long *b;

   266 	char *pool;

   267 

   268 	pool = set->string_pool.data;

   269 	mask = set->buckets.alloc - 1;

   270 	start = hash_string(key) * sizeof(unsigned long);

   271 

   272 	for (i = 0; i < set->buckets.alloc; i += sizeof *b) {

   273 		b = set->buckets.data + ((start + i) & mask);

   274 

   275 		if (*b == 0)

   276 			return 0;

   277 

   278 		if (strcmp(key, &pool[*b]) == 0)

   279 			return *b;

   280 	}

   281 

   282 	return 0;

   283 }

   284 

   285 static unsigned long

   286 add_to_string_pool(struct razor_set *set, const char *key)

   287 {

   288 	int len;

   289 	char *p;

   290 

   291 	len = strlen(key) + 1;

   292 	p = array_add(&set->string_pool, len);

   293 	memcpy(p, key, len);

   294 

   295 	return p - (char *) set->string_pool.data;

   296 }

   297 

   298 static void

   299 do_insert(struct razor_set *set, unsigned long value)

   300 {

   301 	unsigned int mask, start, i;

   302 	unsigned long *b;

   303 	const char *key;

   304 

   305 	key = (char *) set->string_pool.data + value;

   306 	mask = set->buckets.alloc - 1;

   307 	start = hash_string(key) * sizeof(unsigned long);

   308 

   309 	for (i = 0; i < set->buckets.alloc; i += sizeof *b) {

   310 		b = set->buckets.data + ((start + i) & mask);

   311 		if (*b == 0) {

   312 			*b = value;

   313 			break;

   314 		}

   315 	}

   316 }

   317 

   318 unsigned long

   319 razor_set_insert(struct razor_set *set, const char *key)

   320 {

   321 	unsigned long value, *buckets, *b, *end;

   322 	int alloc;

   323 

   324 	alloc = set->buckets.alloc;

   325 	array_add(&set->buckets, 4 * sizeof *buckets);

   326 	if (alloc != set->buckets.alloc) {

   327 		end = set->buckets.data + alloc;

   328 		memset(end, 0, set->buckets.alloc - alloc);

   329 		for (b = set->buckets.data; b < end; b++) {

   330 			value = *b;

   331 			if (value != 0) {

   332 				*b = 0;

   333 				do_insert(set, value);

   334 			}

   335 		}

   336 	}

   337 

   338 	value = add_to_string_pool(set, key);

   339 	do_insert (set, value);

   340 

   341 	return value;

   342 }

   343 

   344 static unsigned long

   345 razor_set_add_package(struct razor_set *set,

   346 		      unsigned long name, unsigned long version)

   347 {

   348 	struct razor_package *p;

   349 

   350 	p = array_add(&set->packages, sizeof *p);

   351 

   352 	p->name = name;

   353 	p->version = version;

   354 

   355 	return p - (struct razor_package *) set->packages.data;

   356 }

   357 

   358 static unsigned long

   359 razor_set_add_provides(struct razor_set *set,

   360 		       unsigned long name, unsigned long version)

   361 {

   362 	struct razor_provides *p;

   363 

   364 	p = array_add(&set->provides, sizeof *p);

   365 

   366 	p->name = name;

   367 	p->version = version;

   368 

   369 	return p - (struct razor_provides *) set->packages.data;

   370 }

   371 

   372 unsigned long

   373 razor_set_tokenize(struct razor_set *set, const char *string)

   374 {

   375 	unsigned long token;

   376 

   377 	token = razor_set_lookup(set, string);

   378 	if (token != 0)

   379 		return token;

   380 

   381 	return razor_set_insert(set, string);

   382 }

   383 

   384 static struct razor_set *qsort_set;

   385 

   386 static int

   387 compare_packages(const void *p1, const void *p2)

   388 {

   389 	const struct razor_package *pkg1 = p1, *pkg2 = p2;

   390 	char *pool = qsort_set->string_pool.data;

   391 

   392 	return strcmp(&pool[pkg1->name], &pool[pkg2->name]);

   393 }

   394 

   395 static int

   396 compare_provides(const void *p1, const void *p2)

   397 {

   398 	const struct razor_provides *prv1 = p1, *prv2 = p2;

   399 	char *pool = qsort_set->string_pool.data;

   400 

   401 	return strcmp(&pool[prv1->name], &pool[prv2->name]);

   402 }

   403 

   404 static void

   405 razor_set_sort(struct razor_set *set)

   406 {

   407 	qsort_set = set;

   408 	qsort(set->packages.data,

   409 	      set->packages.size / sizeof(struct razor_package),

   410 	      sizeof(struct razor_package), compare_packages);

   411 	qsort(set->provides.data,

   412 	      set->provides.size / sizeof(struct razor_provides),

   413 	      sizeof(struct razor_provides), compare_provides);

   414 }

   415 

   416 struct parsing_context {

   417 	struct razor_set *set;

   418 	int pkg_id;

   419 };

   420 

   421 static void

   422 parse_package(struct parsing_context *ctx, const char **atts)

   423 {

   424 	unsigned long name = 0, version = 0;

   425 	int i;

   426 

   427 	for (i = 0; atts[i]; i += 2) {

   428 		if (strcmp(atts[i], "name") == 0)

   429 			name = razor_set_tokenize(ctx->set, atts[i + 1]);

   430 		else if (strcmp(atts[i], "version") == 0)

   431 			version = razor_set_tokenize(ctx->set, atts[i + 1]);

   432 	}

   433 

   434 	if (name == 0 || version == 0) {

   435 		fprintf(stderr, "invalid package tag, "

   436 			"missing name or version attributes\n");

   437 		return;

   438 	}

   439 

   440 	ctx->pkg_id = razor_set_add_package(ctx->set, name, version);

   441 

   442 	return;

   443 }

   444 

   445 static void

   446 parse_provides(struct parsing_context *ctx, const char **atts)

   447 {

   448 	unsigned long name = 0, version = 0;

   449 	int i;

   450 

   451 	for (i = 0; atts[i]; i += 2) {

   452 		if (strcmp(atts[i], "name") == 0)

   453 			name = razor_set_tokenize(ctx->set, atts[i + 1]);

   454 	}

   455 	

   456 	if (name == 0) {

   457 		fprintf(stderr, "invalid provides tag, "

   458 			"missing name attribute\n");

   459 		return;

   460 	}

   461 

   462 	ctx->pkg_id = razor_set_add_provides(ctx->set, name, version);

   463 }

   464 

   465 static void

   466 start_element(void *data, const char *name, const char **atts)

   467 {

   468 	struct parsing_context *ctx = data;

   469 	int i;

   470 

   471 	if (strcmp(name, "package") == 0)

   472 		parse_package(ctx, atts);

   473 	else if (strcmp(name, "provides") == 0)

   474 		parse_provides(ctx, atts);

   475 

   476 	for (i = 0; atts[i]; i += 2)

   477 		razor_set_tokenize(ctx->set, atts[i + 1]);

   478 }

   479 

   480 static void

   481 end_element (void *data, const char *name)

   482 {

   483 	struct parsing_context *ctx = data;

   484 

   485 	if (strcmp(name, "package") == 0)

   486 		ctx->pkg_id = 0;

   487 }

   488 

   489 static char *

   490 sha1_to_hex(const unsigned char *sha1)

   491 {

   492 	static int bufno;

   493 	static char hexbuffer[4][50];

   494 	static const char hex[] = "0123456789abcdef";

   495 	char *buffer = hexbuffer[3 & ++bufno], *buf = buffer;

   496 	int i;

   497 

   498 	for (i = 0; i < 20; i++) {

   499 		unsigned int val = *sha1++;

   500 		*buf++ = hex[val >> 4];

   501 		*buf++ = hex[val & 0xf];

   502 	}

   503 	*buf = '\0';

   504 

   505 	return buffer;

   506 }

   507 

   508 static int

   509 razor_set_import(struct razor_set *set, const char *filename)

   510 {

   511 	SHA_CTX sha1;

   512 	XML_Parser parser;

   513 	struct parsing_context ctx;

   514 	int fd;

   515 	void *p;

   516 	struct stat stat;

   517 	char buf[128];

   518 	unsigned char hash[20];

   519 

   520 	fd = open(filename, O_RDONLY);

   521 	if (fstat(fd, &stat) < 0)

   522 		return -1;

   523 	p = mmap(NULL, stat.st_size, PROT_READ, MAP_PRIVATE, fd, 0);

   524 	if (p == MAP_FAILED)

   525 		return -1;

   526 

   527 	parser = XML_ParserCreate(NULL);

   528 	ctx.set = set;

   529 	XML_SetUserData(parser, &ctx);

   530 	XML_SetElementHandler(parser, start_element, end_element);

   531 	if (XML_Parse(parser, p, stat.st_size, 1) == XML_STATUS_ERROR) {

   532 		fprintf(stderr,

   533 			"%s at line %d, %s\n",

   534 			XML_ErrorString(XML_GetErrorCode(parser)),

   535 			XML_GetCurrentLineNumber(parser),

   536 			filename);

   537 		return 1;

   538 	}

   539 

   540 	XML_ParserFree(parser);

   541 

   542 	SHA1_Init(&sha1);

   543 	SHA1_Update(&sha1, p, stat.st_size);

   544 	SHA1_Final(hash, &sha1);

   545 

   546 	close(fd);

   547 

   548 	snprintf(buf, sizeof buf, "set/%s", sha1_to_hex(hash));

   549 	if (write_to_file(buf, p, stat.st_size) < 0)

   550 		return -1;

   551 	munmap(p, stat.st_size);

   552 

   553 	return 0;

   554 }

   555 

   556 void

   557 razor_set_list(struct razor_set *set)

   558 {

   559 	struct razor_package *p, *end;

   560 	char *pool;

   561 

   562 	pool = set->string_pool.data;

   563 	end = set->packages.data + set->packages.size;

   564 	for (p = set->packages.data; p < end && p->name; p++)

   565 		printf("%s %s\n", &pool[p->name], &pool[p->version]);

   566 }

   567 

   568 void

   569 razor_set_list_provides(struct razor_set *set)

   570 {

   571 	struct razor_provides *p, *end;

   572 	char *pool;

   573 

   574 	pool = set->string_pool.data;

   575 	end = set->provides.data + set->provides.size;

   576 	for (p = set->provides.data; p < end && p->name; p++)

   577 		printf("%s %s\n", &pool[p->name], &pool[p->version]);

   578 }

   579 

   580 void

   581 razor_set_info(struct razor_set *set)

   582 {

   583 	unsigned int offset, size;

   584 	int i;

   585 

   586 	for (i = 0; i < set->header->sections[i].type; i++) {

   587 		offset = set->header->sections[i].offset;

   588 		size = set->header->sections[i + 1].offset - offset;

   589 

   590 		switch (set->header->sections[i].type) {

   591 		case RAZOR_BUCKETS:

   592 			printf("bucket section:\t\t%dkb\n", size / 1024);

   593 			break;

   594 		case RAZOR_STRINGS:

   595 			printf("string pool:\t\t%dkb\n", size / 1024);

   596 			break;

   597 		case RAZOR_PACKAGES:

   598 			printf("package section:\t%dkb\n", size / 1024);

   599 			break;

   600 		case RAZOR_PROVIDES:

   601 			printf("provides section:\t%dkb\n", size / 1024);

   602 			break;

   603 		}

   604 	}

   605 }

   606 

   607 static int

   608 usage(void)

   609 {

   610 	printf("usage: razor [ import FILES | lookup <key> | "

   611 	       "list | list-provides | info ]\n");

   612 	exit(1);

   613 }

   614 

   615 static const char repo_filename[] = "system.repo";

   616 

   617 int

   618 main(int argc, char *argv[])

   619 {

   620 	int i;

   621 	struct razor_set *set;

   622 	struct stat statbuf;

   623 

   624 	if (argc < 2) {

   625 		usage();

   626 	} else if (strcmp(argv[1], "import") == 0) {

   627 		if (stat("set", &statbuf) && mkdir("set", 0777)) {

   628 			fprintf(stderr, "could not create directory 'set'\n");

   629 			exit(-1);

   630 		}

   631 			

   632 		set = razor_set_create();

   633 

   634 		for (i = 2; i < argc; i++) {

   635 			if (razor_set_import(set, argv[i]) < 0) {

   636 				fprintf(stderr, "failed to import %s\n",

   637 					argv[i]);

   638 				exit(-1);

   639 			}

   640 		}

   641 

   642 		razor_set_sort(set);

   643 

   644 		/* FIXME: We add a sentinel package here, but we

   645 		 * should probably just have a size field in the

   646 		 * header section. */

   647 		razor_set_add_package(set, 0, 0);

   648 		razor_set_add_provides(set, 0, 0);

   649 

   650 		printf("bucket allocation: %d\n", set->buckets.alloc);

   651 		printf("pool size: %d\n", set->string_pool.size);

   652 		printf("pool allocation: %d\n", set->string_pool.alloc);

   653 		printf("packages: %d\n",

   654 		       set->packages.size / sizeof(struct razor_package));

   655 		printf("provides: %d\n",

   656 		       set->provides.size / sizeof(struct razor_provides));

   657 

   658 		razor_set_write(set, repo_filename);

   659 

   660 		razor_set_destroy(set);

   661 	} else if (strcmp(argv[1], "lookup") == 0) {

   662 		set = razor_set_open(repo_filename);

   663 		printf("%s is %lu\n", argv[2],

   664 		       razor_set_lookup(set, argv[2]));

   665 		razor_set_destroy(set);

   666 	} else if (strcmp(argv[1], "list") == 0) {

   667 		set = razor_set_open(repo_filename);

   668 		razor_set_list(set);

   669 		razor_set_destroy(set);

   670 	} else if (strcmp(argv[1], "list-provides") == 0) {

   671 		set = razor_set_open(repo_filename);

   672 		razor_set_list_provides(set);

   673 		razor_set_destroy(set);

   674 	} else if (strcmp(argv[1], "info") == 0) {

   675 		set = razor_set_open(repo_filename);

   676 		razor_set_info(set);

   677 		razor_set_destroy(set);

   678 	} else {

   679 		usage();

   680 	}

   681 

   682 	return 0;

   683 }
razor