Mercurialrazor / file revision
summary | shortlog | changelog | graph | tags | branches | files | changeset | file | revisions | annotate | diff | raw
razor.c
author Kristian H?gsberg <krh@redhat.com>
Tue Sep 04 23:52:59 2007 -0400 (2007-09-04)
changeset 6 4eeed5fbe6b7
parent 4 c8e9dbcfaf02
child 7 235f7daf817c
permissions -rw-r--r--
Factor out array code. 
     1 #include <stdlib.h>

     2 #include <stdio.h>

     3 #include <string.h>

     4 #include <sys/types.h>

     5 #include <sys/stat.h>

     6 #include <sys/mman.h>

     7 #include <unistd.h>

     8 #include <fcntl.h>

     9 

    10 #include <expat.h>

    11 #include "sha1.h"

    12 

    13 struct array {

    14 	void *data;

    15 	int size, alloc;

    16 };

    17 

    18 static void *

    19 array_add(struct array *array, int size)

    20 {

    21 	int alloc;

    22 	void *data, *p;

    23 

    24 	if (array->alloc > 0)

    25 		alloc = array->alloc;

    26 	else

    27 		alloc = 1024;

    28 

    29 	while (alloc < array->size + size)

    30 		alloc *= 2;

    31 

    32 	if (array->alloc < alloc) {

    33 		data = realloc(array->data, alloc);

    34 		if (data == NULL)

    35 			return 0;

    36 		array->data = data;

    37 		array->alloc = alloc;

    38 	}

    39 

    40 	p = array->data + array->size;

    41 	array->size += size;

    42 

    43 	return p;

    44 }

    45 

    46 static int

    47 write_to_fd(int fd, void *p, size_t size)

    48 {

    49 	int rest, len;

    50 

    51 	rest = size;

    52 	while (rest > 0) {

    53 		len = write(fd, p, rest);

    54 		if (len < 0)

    55 			return -1;

    56 		rest -= len;

    57 	}

    58 

    59 	return 0;

    60 }

    61 

    62 static int

    63 write_to_file(const char *filename, void *p, size_t size)

    64 {

    65 	int fd, err;

    66 

    67 	fd = open(filename, O_CREAT | O_WRONLY | O_TRUNC, 0666);

    68 	if (fd < 0)

    69 		return -1;

    70 	err = write_to_fd(fd, p, size);

    71 	close(fd);

    72 

    73 	return err;

    74 }

    75 

    76 static void *

    77 zalloc(size_t size)

    78 {

    79 	void *p;

    80 

    81 	p = malloc(size);

    82 	memset(p, 0, size);

    83 

    84 	return p;

    85 }

    86 

    87 struct razor_set_header {

    88 	unsigned int magic;

    89 	unsigned int version;

    90 	struct { unsigned int type, offset; } sections[0];

    91 };

    92 

    93 #define RAZOR_MAGIC 0x7a7a7a7a

    94 #define RAZOR_VERSION 1

    95 #define RAZOR_BUCKETS 1

    96 #define RAZOR_STRINGS 2

    97 #define RAZOR_PACKAGES 3

    98 

    99 struct razor_package {

   100 	unsigned long name;

   101 	unsigned long version;

   102 };

   103 

   104 struct razor_set {

   105 	struct array buckets;

   106 	struct array string_pool;

   107 	struct array packages;

   108 	struct razor_set_header *header;

   109 };

   110 

   111 struct razor_set *

   112 razor_set_create(void)

   113 {

   114 	struct razor_set *set;

   115 	char *p;

   116 

   117 	set = zalloc(sizeof(struct razor_set));

   118 	p = array_add(&set->string_pool, 1);

   119 	*p = '\0';

   120 

   121 	return set;

   122 }

   123 

   124 struct razor_set *

   125 razor_set_open(const char *filename)

   126 {

   127 	struct razor_set *set;

   128 	struct stat stat;

   129 	unsigned int size, offset;

   130 	int fd, i;

   131 

   132 	set = zalloc(sizeof *set);

   133 	fd = open(filename, O_RDONLY);

   134 	if (fstat(fd, &stat) < 0)

   135 		return NULL;

   136 	set->header = mmap(NULL, stat.st_size, PROT_READ, MAP_PRIVATE, fd, 0);

   137 	if (set->header == MAP_FAILED) {

   138 		free(set);

   139 		return NULL;

   140 	}

   141 

   142 	for (i = 0; i < set->header->sections[i].type; i++) {

   143 		offset = set->header->sections[i].offset;

   144 		size = set->header->sections[i + 1].offset - offset;

   145 

   146 		switch (set->header->sections[i].type) {

   147 		case RAZOR_BUCKETS:

   148 			set->buckets.data = (void *) set->header + offset;

   149 			set->buckets.size = size;

   150 			set->buckets.alloc = size;

   151 			break;

   152 		case RAZOR_STRINGS:

   153 			set->string_pool.data = (void *) set->header + offset;

   154 			set->string_pool.size = size;

   155 			set->string_pool.alloc = size;

   156 			break;

   157 		case RAZOR_PACKAGES:

   158 			set->packages.data = (void *) set->header + offset;

   159 			set->packages.size = size;

   160 			set->packages.size = size;

   161 			break;

   162 		}

   163 	}

   164 	close(fd);

   165 

   166 	return set;

   167 }

   168 

   169 void

   170 razor_set_destroy(struct razor_set *set)

   171 {

   172 	unsigned int size;

   173 	int i;

   174 

   175 	if (set->header) {

   176 		for (i = 0; set->header->sections[i].type; i++)

   177 			;

   178 		size = set->header->sections[i].type;

   179 		munmap(set->header, size);

   180 	} else {

   181 		free(set->buckets.data);

   182 		free(set->string_pool.data);

   183 		free(set->packages.data);

   184 	}

   185 

   186 	free(set);

   187 }

   188 

   189 static int

   190 razor_set_write(struct razor_set *set, const char *filename)

   191 {

   192 	char data[4096];

   193 	struct razor_set_header *header = (struct razor_set_header *) data;

   194 	int fd, pool_size, packages_size;

   195 

   196 	/* Align these to pages sizes */

   197 	pool_size = (set->string_pool.size + 4095) & ~4095;

   198 	packages_size = (set->packages.size + 4095) & ~4095;

   199 

   200 	memset(data, 0, sizeof data);

   201 	header->magic = RAZOR_MAGIC;

   202 	header->version = RAZOR_VERSION;

   203 

   204 	header->sections[0].type = RAZOR_BUCKETS;

   205 	header->sections[0].offset = sizeof data;

   206 

   207 	header->sections[1].type = RAZOR_STRINGS;

   208 	header->sections[1].offset =

   209 		header->sections[0].offset + set->buckets.alloc;

   210 

   211 	header->sections[2].type = RAZOR_PACKAGES;

   212 	header->sections[2].offset =

   213 		header->sections[1].offset + pool_size;

   214 

   215 	header->sections[3].type = 0;

   216 	header->sections[3].offset =

   217 		header->sections[2].offset + packages_size;

   218 

   219 	fd = open(filename, O_CREAT | O_WRONLY | O_TRUNC, 0666);

   220 	if (fd < 0)

   221 		return -1;

   222 

   223 	write_to_fd(fd, data, sizeof data);

   224 	write_to_fd(fd, set->buckets.data, set->buckets.alloc);

   225 	write_to_fd(fd, set->string_pool.data, pool_size);

   226 	write_to_fd(fd, set->packages.data, packages_size);

   227 

   228 	return 0;

   229 }

   230 

   231 static unsigned int

   232 hash_string(const char *key)

   233 {

   234 	const char *p;

   235 	unsigned int hash = 0;

   236 

   237 	for (p = key; *p; p++)

   238 		hash = (hash << 2) ^ *p;

   239 

   240 	return hash;

   241 }

   242 

   243 unsigned long

   244 razor_set_lookup(struct razor_set *set, const char *key)

   245 {

   246 	unsigned int mask, start, i;

   247 	unsigned long *b;

   248 	char *pool;

   249 

   250 	pool = set->string_pool.data;

   251 	mask = set->buckets.alloc - 1;

   252 	start = hash_string(key) * sizeof(unsigned long);

   253 

   254 	for (i = 0; i < set->buckets.alloc; i += sizeof *b) {

   255 		b = set->buckets.data + ((start + i) & mask);

   256 

   257 		if (*b == 0)

   258 			return 0;

   259 

   260 		if (strcmp(key, &pool[*b]) == 0)

   261 			return *b;

   262 	}

   263 

   264 	return 0;

   265 }

   266 

   267 static unsigned long

   268 add_to_string_pool(struct razor_set *set, const char *key)

   269 {

   270 	int len;

   271 	char *p;

   272 

   273 	len = strlen(key) + 1;

   274 	p = array_add(&set->string_pool, len);

   275 	memcpy(p, key, len);

   276 

   277 	return p - (char *) set->string_pool.data;

   278 }

   279 

   280 static void

   281 do_insert(struct razor_set *set, unsigned long value)

   282 {

   283 	unsigned int mask, start, i;

   284 	unsigned long *b;

   285 	const char *key;

   286 

   287 	key = (char *) set->string_pool.data + value;

   288 	mask = set->buckets.alloc - 1;

   289 	start = hash_string(key) * sizeof(unsigned long);

   290 

   291 	for (i = 0; i < set->buckets.alloc; i += sizeof *b) {

   292 		b = set->buckets.data + ((start + i) & mask);

   293 		if (*b == 0) {

   294 			*b = value;

   295 			break;

   296 		}

   297 	}

   298 }

   299 

   300 unsigned long

   301 razor_set_insert(struct razor_set *set, const char *key)

   302 {

   303 	unsigned long value, *buckets, *b, *end;

   304 	int alloc;

   305 

   306 	alloc = set->buckets.alloc;

   307 	array_add(&set->buckets, 4 * sizeof *buckets);

   308 	if (alloc != set->buckets.alloc) {

   309 		end = set->buckets.data + alloc;

   310 		memset(end, 0, set->buckets.alloc - alloc);

   311 		for (b = set->buckets.data; b < end; b++) {

   312 			value = *b;

   313 			if (value != 0) {

   314 				*b = 0;

   315 				do_insert(set, value);

   316 			}

   317 		}

   318 	}

   319 

   320 	value = add_to_string_pool(set, key);

   321 	do_insert (set, value);

   322 

   323 	return value;

   324 }

   325 

   326 static unsigned long

   327 razor_set_add_package(struct razor_set *set,

   328 		      unsigned long name, unsigned long version)

   329 {

   330 	struct razor_package *p;

   331 

   332 	p = array_add(&set->packages, sizeof *p);

   333 

   334 	p->name = name;

   335 	p->version = version;

   336 

   337 	return p - (struct razor_package *) set->packages.data;

   338 }

   339 

   340 unsigned long

   341 razor_set_tokenize(struct razor_set *set, const char *string)

   342 {

   343 	unsigned long token;

   344 

   345 	token = razor_set_lookup(set, string);

   346 	if (token != 0)

   347 		return token;

   348 

   349 	return razor_set_insert(set, string);

   350 }

   351 

   352 static struct razor_set *qsort_set;

   353 

   354 static int

   355 compare_packages(const void *p1, const void *p2)

   356 {

   357 	const struct razor_package *pkg1 = p1, *pkg2 = p2;

   358 	char *pool = qsort_set->string_pool.data;

   359 

   360 	return strcmp(&pool[pkg1->name], &pool[pkg2->name]);

   361 }

   362 

   363 static void

   364 razor_set_sort(struct razor_set *set)

   365 {

   366 	qsort_set = set;

   367 	qsort(set->packages.data,

   368 	      set->packages.size / sizeof(struct razor_package),

   369 	      sizeof(struct razor_package), compare_packages);

   370 }

   371 

   372 struct parsing_context {

   373 	struct razor_set *set;

   374 	int pkg_id;

   375 };

   376 

   377 static void

   378 parse_package(struct parsing_context *ctx, const char **atts)

   379 {

   380 	unsigned long name, version;

   381 	int i;

   382 

   383 	for (i = 0; atts[i]; i += 2) {

   384 		if (strcmp(atts[i], "name") == 0)

   385 			name = razor_set_tokenize(ctx->set, atts[i + 1]);

   386 		else if (strcmp(atts[i], "version") == 0)

   387 			version = razor_set_tokenize(ctx->set, atts[i + 1]);

   388 	}

   389 

   390 	if (name == 0 || version == 0) {

   391 		fprintf(stderr, "invalid package tag, "

   392 			"missing name or version attributes\n");

   393 		return;

   394 	}

   395 

   396 	ctx->pkg_id = razor_set_add_package(ctx->set, name, version);

   397 

   398 	return;

   399 }

   400 

   401 static void

   402 start_element(void *data, const char *name, const char **atts)

   403 {

   404 	struct parsing_context *ctx = data;

   405 	int i;

   406 

   407 	if (strcmp(name, "package") == 0)

   408 		parse_package(ctx, atts);

   409 

   410 	for (i = 0; atts[i]; i += 2)

   411 		razor_set_tokenize(ctx->set, atts[i + 1]);

   412 }

   413 

   414 static void

   415 end_element (void *data, const char *name)

   416 {

   417 	struct parsing_context *ctx = data;

   418 

   419 	if (strcmp(name, "package") == 0)

   420 		ctx->pkg_id = 0;

   421 }

   422 

   423 static char *

   424 sha1_to_hex(const unsigned char *sha1)

   425 {

   426 	static int bufno;

   427 	static char hexbuffer[4][50];

   428 	static const char hex[] = "0123456789abcdef";

   429 	char *buffer = hexbuffer[3 & ++bufno], *buf = buffer;

   430 	int i;

   431 

   432 	for (i = 0; i < 20; i++) {

   433 		unsigned int val = *sha1++;

   434 		*buf++ = hex[val >> 4];

   435 		*buf++ = hex[val & 0xf];

   436 	}

   437 	*buf = '\0';

   438 

   439 	return buffer;

   440 }

   441 

   442 static int

   443 razor_set_import(struct razor_set *set, const char *filename)

   444 {

   445 	SHA_CTX sha1;

   446 	XML_Parser parser;

   447 	struct parsing_context ctx;

   448 	int fd;

   449 	void *p;

   450 	struct stat stat;

   451 	char buf[128];

   452 	unsigned char hash[20];

   453 

   454 	fd = open(filename, O_RDONLY);

   455 	if (fstat(fd, &stat) < 0)

   456 		return -1;

   457 	p = mmap(NULL, stat.st_size, PROT_READ, MAP_PRIVATE, fd, 0);

   458 	if (p == MAP_FAILED)

   459 		return -1;

   460 

   461 	parser = XML_ParserCreate(NULL);

   462 	ctx.set = set;

   463 	XML_SetUserData(parser, &ctx);

   464 	XML_SetElementHandler(parser, start_element, end_element);

   465 	if (XML_Parse(parser, p, stat.st_size, 1) == XML_STATUS_ERROR) {

   466 		fprintf(stderr,

   467 			"%s at line %d, %s\n",

   468 			XML_ErrorString(XML_GetErrorCode(parser)),

   469 			XML_GetCurrentLineNumber(parser),

   470 			filename);

   471 		return 1;

   472 	}

   473 

   474 	XML_ParserFree(parser);

   475 

   476 	SHA1_Init(&sha1);

   477 	SHA1_Update(&sha1, p, stat.st_size);

   478 	SHA1_Final(hash, &sha1);

   479 

   480 	close(fd);

   481 

   482 	snprintf(buf, sizeof buf, "set/%s", sha1_to_hex(hash));

   483 	if (write_to_file(buf, p, stat.st_size) < 0)

   484 		return -1;

   485 	munmap(p, stat.st_size);

   486 

   487 	return 0;

   488 }

   489 

   490 void

   491 razor_set_list(struct razor_set *set)

   492 {

   493 	struct razor_package *p, *end;

   494 	char *pool;

   495 

   496 	pool = set->string_pool.data;

   497 	end = set->packages.data + set->packages.size;

   498 	for (p = set->packages.data; p < end && p->name; p++)

   499 		printf("%s %s\n", &pool[p->name], &pool[p->version]);

   500 }

   501 

   502 void

   503 razor_set_info(struct razor_set *set)

   504 {

   505 	unsigned int offset, size;

   506 	int i;

   507 

   508 	for (i = 0; i < set->header->sections[i].type; i++) {

   509 		offset = set->header->sections[i].offset;

   510 		size = set->header->sections[i + 1].offset - offset;

   511 

   512 		switch (set->header->sections[i].type) {

   513 		case RAZOR_BUCKETS:

   514 			printf("bucket section:\t\t%dkb\n", size / 1024);

   515 			break;

   516 		case RAZOR_STRINGS:

   517 			printf("string pool:\t\t%dkb\n", size / 1024);

   518 			break;

   519 		case RAZOR_PACKAGES:

   520 			printf("package section:\t%dkb\n", size / 1024);

   521 			break;

   522 		}

   523 	}

   524 }

   525 

   526 static int

   527 usage(void)

   528 {

   529 	printf("usage: razor [ import FILES | lookup <key> | list | info ]\n");

   530 	exit(1);

   531 }

   532 

   533 static const char repo_filename[] = "system.repo";

   534 

   535 int

   536 main(int argc, char *argv[])

   537 {

   538 	int i;

   539 	struct razor_set *set;

   540 	struct stat statbuf;

   541 

   542 	if (argc < 2) {

   543 		usage();

   544 	} else if (strcmp(argv[1], "import") == 0) {

   545 		if (stat("set", &statbuf) && mkdir("set", 0777)) {

   546 			fprintf(stderr, "could not create directory 'set'\n");

   547 			exit(-1);

   548 		}

   549 			

   550 		set = razor_set_create();

   551 

   552 		for (i = 2; i < argc; i++) {

   553 			if (razor_set_import(set, argv[i]) < 0) {

   554 				fprintf(stderr, "failed to import %s\n",

   555 					argv[i]);

   556 				exit(-1);

   557 			}

   558 		}

   559 

   560 		razor_set_sort(set);

   561 

   562 		/* FIXME: We add a sentinel package here, but we

   563 		 * should probably just have a size field in the

   564 		 * header section. */

   565 		razor_set_add_package(set, 0, 0);

   566 

   567 		printf("bucket allocation: %d\n", set->buckets.alloc);

   568 		printf("pool size: %d\n", set->string_pool.size);

   569 		printf("pool allocation: %d\n", set->string_pool.alloc);

   570 

   571 		razor_set_write(set, repo_filename);

   572 

   573 		razor_set_destroy(set);

   574 	} else if (strcmp(argv[1], "lookup") == 0) {

   575 		set = razor_set_open(repo_filename);

   576 		printf("%s is %lu\n", argv[2],

   577 		       razor_set_lookup(set, argv[2]));

   578 		razor_set_destroy(set);

   579 	} else if (strcmp(argv[1], "list") == 0) {

   580 		set = razor_set_open(repo_filename);

   581 		razor_set_list(set);

   582 		razor_set_destroy(set);

   583 	} else if (strcmp(argv[1], "info") == 0) {

   584 		set = razor_set_open(repo_filename);

   585 		razor_set_info(set);

   586 		razor_set_destroy(set);

   587 	} else {

   588 		usage();

   589 	}

   590 

   591 	return 0;

   592 }
razor