librazor/merger.c
author J. Ali Harlow <ali@juiblex.co.uk>
Thu Oct 09 17:27:41 2014 +0100 (2014-10-09)
changeset 455 df914f383f5c
parent 403 e63951c1d0f8
child 458 3f841a46eab5
permissions -rw-r--r--
Support downloading from local repository even without libcurl

Using the --url option of the razor executable, it is possible
to specify a yum repository on the local machine (eg., on installation
media) and import from there, eg.,:

C> razor --url file:///d:/ import-yum

This will be handled by libcurl if available but if not, an internal
copy routine will be used.

Note that if Microsoft's KTM implementation of atomic transactions is
used, then the current directory must support atomic transactions
(also improve error messages for this, and other, cases).
     1 /*
     2  * Copyright (C) 2008  Kristian Høgsberg <krh@redhat.com>
     3  * Copyright (C) 2008  Red Hat, Inc
     4  * Copyright (C) 2009-2011  J. Ali Harlow <ali@juiblex.co.uk>
     5  *
     6  * This program is free software; you can redistribute it and/or modify
     7  * it under the terms of the GNU General Public License as published by
     8  * the Free Software Foundation; either version 2 of the License, or
     9  * (at your option) any later version.
    10  *
    11  * This program is distributed in the hope that it will be useful,
    12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
    13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    14  * GNU General Public License for more details.
    15  *
    16  * You should have received a copy of the GNU General Public License along
    17  * with this program; if not, write to the Free Software Foundation, Inc.,
    18  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
    19  */
    20 
    21 #include "config.h"
    22 #include <string.h>
    23 #include <assert.h>
    24 #include "razor-internal.h"
    25 #include "razor.h"
    26 
    27 #define UPSTREAM_SOURCE 0x80
    28 
    29 struct source {
    30 	struct razor_set *set;
    31 	uint32_t *property_map;
    32 	uint32_t *file_map;
    33 };
    34 
    35 struct razor_merger {
    36 	int committed;
    37 	struct razor_set *set;
    38 	struct hashtable table;
    39 	struct hashtable file_table;
    40 	struct hashtable details_table;
    41 	struct source source1;
    42 	struct source source2;
    43 };
    44 
    45 struct razor_merger *
    46 razor_merger_create(struct razor_set *set1, struct razor_set *set2)
    47 {
    48 	struct razor_merger *merger;
    49 	int count;
    50 	size_t size;
    51 	uint32_t header_version;
    52 
    53 	merger = zalloc(sizeof *merger);
    54 	merger->set = razor_set_create();
    55 	if (set1->packages.size) {
    56 		header_version = razor_set_get_header_version(set1);
    57 		if (set2->packages.size &&
    58 		    razor_set_get_header_version(set2) > header_version)
    59 			header_version = razor_set_get_header_version(set2);
    60 	} else
    61 		header_version = razor_set_get_header_version(set2);
    62 	razor_set_set_header_version(merger->set, header_version);
    63 	hashtable_init(&merger->table, &merger->set->string_pool);
    64 	hashtable_init(&merger->file_table, &merger->set->file_string_pool);
    65 	hashtable_init(&merger->details_table,
    66 		       &merger->set->details_string_pool);
    67 
    68 	merger->source1.set = set1;
    69 	count = set1->properties.size / sizeof (struct razor_property);
    70 	size = count * sizeof merger->source1.property_map[0];
    71 	merger->source1.property_map = zalloc(size);
    72 	count = set1->files.size / sizeof (struct razor_entry);
    73 	size = count * sizeof merger->source1.file_map[0];
    74 	merger->source1.file_map = zalloc(size);
    75 
    76 	merger->source2.set = set2;
    77 	count = set2->properties.size / sizeof (struct razor_property);
    78 	size = count * sizeof merger->source2.property_map[0];
    79 	merger->source2.property_map = zalloc(size);
    80 	count = set2->files.size / sizeof (struct razor_entry);
    81 	size = count * sizeof merger->source2.file_map[0];
    82 	merger->source2.file_map = zalloc(size);
    83 
    84 	return merger;
    85 }
    86 
    87 void
    88 razor_merger_add_package(struct razor_merger *merger,
    89 			 struct razor_package *package)
    90 {
    91 	char *pool, *details_pool, *s;
    92 	struct list *r;
    93 	struct razor_package *p;
    94 	struct razor_set *set1;
    95 	struct source *source;
    96 	uint32_t flags, *prefix;
    97 	struct array install_prefixes;
    98 
    99 	assert(merger->committed == 0);
   100 
   101 	set1 = merger->source1.set;
   102 	if (set1->packages.data <= (void *) package &&
   103 	    (void *) package < set1->packages.data + set1->packages.size) {
   104 		source = &merger->source1;
   105 		flags = 0;
   106 	} else {
   107 		source = &merger->source2;
   108 		flags = UPSTREAM_SOURCE;
   109 	}
   110 
   111 	pool = source->set->string_pool.data;
   112 	details_pool = source->set->details_string_pool.data;
   113 	p = array_add(&merger->set->packages, sizeof *p);
   114 	p->name = hashtable_tokenize(&merger->table, &pool[package->name]);
   115 	p->flags = flags;
   116 	p->version = hashtable_tokenize(&merger->table,
   117 					&pool[package->version]);
   118 	p->arch = hashtable_tokenize(&merger->table,
   119 				     &pool[package->arch]);
   120 	if (source->set->details_string_pool.size) {
   121 		p->summary = hashtable_tokenize(&merger->details_table,
   122 						&details_pool[package->summary]);
   123 		p->description = hashtable_tokenize(&merger->details_table,
   124 						    &details_pool[package->description]);
   125 		p->url = hashtable_tokenize(&merger->details_table,
   126 					    &details_pool[package->url]);
   127 		p->license = hashtable_tokenize(&merger->details_table,
   128 						&details_pool[package->license]);
   129 	} else {
   130 		p->summary = hashtable_tokenize(&merger->details_table, "");
   131 		p->description = hashtable_tokenize(&merger->details_table, "");
   132 		p->url = hashtable_tokenize(&merger->details_table, "");
   133 		p->license = hashtable_tokenize(&merger->details_table, "");
   134 	}
   135 
   136 	p->properties = package->properties;
   137 	r = list_first(&package->properties, &source->set->property_pool);
   138 	while (r) {
   139 		source->property_map[r->data] = 1;
   140 		r = list_next(r);
   141 	}
   142 
   143 	p->files = package->files;
   144 	r = list_first(&package->files, &source->set->file_pool);
   145 	while (r) {
   146 		source->file_map[r->data] = 1;
   147 		r = list_next(r);
   148 	}
   149 
   150 	array_init(&install_prefixes);
   151 	r = list_first(&package->install_prefixes, &source->set->prefix_pool);
   152 	while (r) {
   153 		s = (char *)source->set->string_pool.data + r->data;
   154 		prefix = array_add(&install_prefixes, sizeof *prefix);
   155 		*prefix = hashtable_tokenize(&merger->table, s);
   156 		r = list_next(r);
   157 	}
   158 	list_set_array(&p->install_prefixes, &merger->set->prefix_pool,
   159                        &install_prefixes, 0);
   160 	array_release(&install_prefixes);
   161 
   162 	p->preun.program = hashtable_tokenize(&merger->table,
   163 					      &pool[package->preun.program]);
   164 	p->preun.body = hashtable_tokenize(&merger->table,
   165 					   &pool[package->preun.body]);
   166 	p->postun.program = hashtable_tokenize(&merger->table,
   167 					       &pool[package->postun.program]);
   168 	p->postun.body = hashtable_tokenize(&merger->table,
   169 					    &pool[package->postun.body]);
   170 }
   171 
   172 static uint32_t
   173 add_property(struct razor_merger *merger,
   174 	     const char *name, uint32_t flags, const char *version)
   175 {
   176 	struct razor_property *p;
   177 
   178 	p = array_add(&merger->set->properties, sizeof *p);
   179 	p->name = hashtable_tokenize(&merger->table, name);
   180 	p->flags = flags;
   181 	p->version = hashtable_tokenize(&merger->table, version);
   182 
   183 	return p - (struct razor_property *) merger->set->properties.data;
   184 }
   185 
   186 static void
   187 merge_properties(struct razor_merger *merger)
   188 {
   189 	struct razor_property *p1, *p2;
   190 	struct razor_set *set1, *set2;
   191 	uint32_t *map1, *map2;
   192 	int i, j, cmp, count1, count2;
   193 	char *pool1, *pool2;
   194 
   195 	set1 = merger->source1.set;
   196 	set2 = merger->source2.set;
   197 	map1 = merger->source1.property_map;
   198 	map2 = merger->source2.property_map;
   199 
   200 	i = 0;
   201 	j = 0;
   202 	pool1 = set1->string_pool.data;
   203 	pool2 = set2->string_pool.data;
   204 
   205 	count1 = set1->properties.size / sizeof *p1;
   206 	count2 = set2->properties.size / sizeof *p2;
   207 	while (i < count1 || j < count2) {
   208 		if (i < count1 && map1[i] == 0) {
   209 			i++;
   210 			continue;
   211 		}
   212 		if (j < count2 && map2[j] == 0) {
   213 			j++;
   214 			continue;
   215 		}
   216 		p1 = (struct razor_property *) set1->properties.data + i;
   217 		p2 = (struct razor_property *) set2->properties.data + j;
   218 		if (i < count1 && j < count2)
   219 			cmp = strcmp(&pool1[p1->name], &pool2[p2->name]);
   220 		else if (i < count1)
   221 			cmp = -1;
   222 		else
   223 			cmp = 1;
   224 		if (cmp == 0)
   225 			cmp = p1->flags - p2->flags;
   226 		if (cmp == 0)
   227 			cmp = razor_versioncmp(&pool1[p1->version],
   228 					       &pool2[p2->version]);
   229 		if (cmp < 0) {
   230 			map1[i++] = add_property(merger,
   231 						 &pool1[p1->name],
   232 						 p1->flags,
   233 						 &pool1[p1->version]);
   234 		} else if (cmp > 0) {
   235 			map2[j++] = add_property(merger,
   236 						 &pool2[p2->name],
   237 						 p2->flags,
   238 						 &pool2[p2->version]);
   239 		} else  {
   240 			map1[i++] = map2[j++] =
   241 				add_property(merger,
   242 					     &pool1[p1->name],
   243 					     p1->flags,
   244 					     &pool1[p1->version]);
   245 		}
   246 	}
   247 }
   248 
   249 static void
   250 emit_properties(struct list_head *properties, struct array *source_pool,
   251 		uint32_t *map, struct array *pool)
   252 {
   253 	uint32_t r;
   254 	struct list *p, *q;
   255 
   256 	r = pool->size / sizeof *q;
   257 	p = list_first(properties, source_pool);
   258 	while (p) {
   259 		q = array_add(pool, sizeof *q);
   260 		q->data = map[p->data];
   261 		q->flags = p->flags;
   262 		p = list_next(p);
   263 	}
   264 
   265 	list_set_ptr(properties, r);
   266 }
   267 
   268 static uint32_t
   269 add_file(struct razor_merger *merger, const char *name)
   270 {
   271 	struct razor_entry *e;
   272 
   273 	e = array_add(&merger->set->files, sizeof *e);
   274 	e->name = hashtable_tokenize(&merger->file_table, name);
   275 	e->flags = 0;
   276 	e->start = 0;
   277 
   278 	return e - (struct razor_entry *)merger->set->files.data;
   279 }
   280 
   281 /* FIXME. Blah */
   282 static int
   283 fix_file_map(uint32_t *map,
   284 	     struct razor_entry *files,
   285 	     struct razor_entry *top)
   286 {
   287 	uint32_t e;
   288 	int found_file = 0;
   289 
   290 	e = top - files;
   291 	do {
   292 		if (files[e].start &&
   293 		    fix_file_map(map, files, &files[files[e].start]))
   294 			map[e] = 1;
   295 		if (map[e])
   296 			found_file = 1;
   297 	} while (!(files[e++].flags & RAZOR_ENTRY_LAST));
   298 
   299 	return found_file;
   300 }
   301 
   302 struct merge_directory {
   303 	uint32_t merged, dir1, dir2;
   304 };
   305 
   306 static void
   307 merge_one_directory(struct razor_merger *merger, struct merge_directory *md)
   308 {
   309 	struct razor_entry *root1, *root2, *mroot, *e1, *e2;
   310 	struct razor_set *set1, *set2;
   311 	struct array merge_stack;
   312 	struct merge_directory *child_md, *end_md;
   313 	uint32_t *map1, *map2, start, last;
   314 	int cmp;
   315 	char *pool1, *pool2;
   316 
   317 	set1 = merger->source1.set;
   318 	set2 = merger->source2.set;
   319 	map1 = merger->source1.file_map;
   320 	map2 = merger->source2.file_map;
   321 	pool1 = set1->file_string_pool.data;
   322 	pool2 = set2->file_string_pool.data;
   323 	root1 = (struct razor_entry *) set1->files.data;
   324 	root2 = (struct razor_entry *) set2->files.data;
   325 
   326 	array_init(&merge_stack);
   327 
   328 	start = merger->set->files.size / sizeof (struct razor_entry);
   329 	last = 0xFFFFFFFF;
   330 	e1 = md->dir1 != 0xFFFFFFFF ? root1 + md->dir1 : NULL;
   331 	e2 = md->dir2 != 0xFFFFFFFF ? root2 + md->dir2 : NULL;
   332 	while (e1 || e2) {
   333 		if (!e2 && !map1[e1 - root1]) {
   334 			if ((e1++)->flags & RAZOR_ENTRY_LAST)
   335 				e1 = NULL;
   336 			continue;
   337 		}
   338 		if (!e1 && !map2[e2 - root2]) {
   339 			if ((e2++)->flags & RAZOR_ENTRY_LAST)
   340 				e2 = NULL;
   341 			continue;
   342 		}
   343 		if (e1 && !map1[e1 - root1] &&
   344 		    e2 && !map2[e2 - root2]) {
   345 			if ((e1++)->flags & RAZOR_ENTRY_LAST)
   346 				e1 = NULL;
   347 			if ((e2++)->flags & RAZOR_ENTRY_LAST)
   348 				e2 = NULL;
   349 			continue;
   350 		}
   351 
   352 		if (!e1)
   353 			cmp = 1;
   354 		else if (!e2)
   355 			cmp = -1;
   356 		else {
   357 			cmp = strcmp (&pool1[e1->name],
   358 				      &pool2[e2->name]);
   359 		}
   360 
   361 		if (cmp < 0) {
   362 			if (map1[e1 - root1]) {
   363 				map1[e1 - root1] = last =
   364 					add_file(merger, &pool1[e1->name]);
   365 				if (e1->start) {
   366 					child_md = array_add(&merge_stack, sizeof (struct merge_directory));
   367 					child_md->merged = last;
   368 					child_md->dir1 = e1->start;
   369 					child_md->dir2 = 0xFFFFFFFF;
   370 				}
   371 			}
   372 			if ((e1++)->flags & RAZOR_ENTRY_LAST)
   373 				e1 = NULL;
   374 		} else if (cmp > 0) {
   375 			if (map2[e2 - root2]) {
   376 				map2[e2 - root2] = last =
   377 					add_file(merger, &pool2[e2->name]);
   378 				if (e2->start) {
   379 					child_md = array_add(&merge_stack, sizeof (struct merge_directory));
   380 					child_md->merged = last;
   381 					child_md->dir1 = 0xFFFFFFFF;
   382 					child_md->dir2 = e2->start;
   383 				}
   384 			}
   385 			if ((e2++)->flags & RAZOR_ENTRY_LAST)
   386 				e2 = NULL;
   387 		} else {
   388 			map1[e1 - root1] = map2[e2- root2] = last =
   389 				add_file(merger, &pool1[e1->name]);
   390 			if (e1->start || e2->start) {
   391 				child_md = array_add(&merge_stack, sizeof (struct merge_directory));
   392 				child_md->merged = last;
   393 				child_md->dir1 = e1->start ? e1->start : 0xFFFFFFFF;
   394 				child_md->dir2 = e2->start ? e2->start : 0xFFFFFFFF;
   395 			}
   396 			if ((e1++)->flags & RAZOR_ENTRY_LAST)
   397 				e1 = NULL;
   398 			if ((e2++)->flags & RAZOR_ENTRY_LAST)
   399 				e2 = NULL;
   400 		}
   401 	}
   402 
   403 	mroot = (struct razor_entry *)merger->set->files.data;
   404 	if (last != 0xFFFFFFFF) {
   405 		mroot[last].flags = RAZOR_ENTRY_LAST;
   406 		mroot[md->merged].start = start;
   407 	}
   408 
   409 	end_md = merge_stack.data + merge_stack.size;
   410 	for (child_md = merge_stack.data; child_md < end_md; child_md++)
   411 		merge_one_directory(merger, child_md);
   412 	array_release(&merge_stack);
   413 }
   414 
   415 static void
   416 merge_files(struct razor_merger *merger)
   417 {
   418 	struct razor_entry *root;
   419 	struct merge_directory md;
   420 	uint32_t *map1, *map2;
   421 
   422 	map1 = merger->source1.file_map;
   423 	map2 = merger->source2.file_map;
   424 
   425 	md.merged = 0;
   426 
   427 	if (merger->source1.set->files.size) {
   428 		root = (struct razor_entry *) merger->source1.set->files.data;
   429 		if (root->start)
   430 			fix_file_map(map1, root, root);
   431 		md.dir1 = 0;
   432 	} else {
   433 		md.dir1 = 0xFFFFFFFF;
   434 	}
   435 
   436 	if (merger->source2.set->files.size) {
   437 		root = (struct razor_entry *) merger->source2.set->files.data;
   438 		if (root->start)
   439 			fix_file_map(map2, root, root);
   440 		md.dir2 = 0;
   441 	} else {
   442 		md.dir2 = 0xFFFFFFFF;
   443 	}
   444 
   445 	/* Remove the unnamed root which razor_set_create() added */
   446 	array_release(&merger->set->files);
   447 	array_init(&merger->set->files);
   448 
   449 	merge_one_directory(merger, &md);
   450 }
   451 
   452 static void
   453 emit_files(struct list_head *files, struct array *source_pool,
   454 	   uint32_t *map, struct array *pool)
   455 {
   456 	uint32_t r;
   457 	struct list *p, *q;
   458 
   459 	r = pool->size / sizeof *q;
   460 	p = list_first(files, source_pool);
   461 	while (p) {
   462 		q = array_add(pool, sizeof *q);
   463 		q->data = map[p->data];
   464 		q->flags = p->flags;
   465 		p = list_next(p);
   466 	}
   467 
   468 	list_set_ptr(files, r);
   469 }
   470 
   471 /* Rebuild property->packages maps.  We can't just remap these, as a
   472  * property may have lost or gained a number of packages.  Allocate an
   473  * array per property and loop through the packages and add them to
   474  * the arrays for their properties. */
   475 static void
   476 rebuild_property_package_lists(struct razor_set *set)
   477 {
   478 	struct array *pkgs, *a;
   479 	struct razor_package *pkg, *pkg_end;
   480 	struct razor_property *prop, *prop_end;
   481 	struct list *r;
   482 	uint32_t *q;
   483 	int count;
   484 
   485 	count = set->properties.size / sizeof (struct razor_property);
   486 	pkgs = zalloc(count * sizeof *pkgs);
   487 	pkg_end = set->packages.data + set->packages.size;
   488 
   489 	for (pkg = set->packages.data; pkg < pkg_end; pkg++) {
   490 		r = list_first(&pkg->properties, &set->property_pool);
   491 		while (r) {
   492 			q = array_add(&pkgs[r->data], sizeof *q);
   493 			*q = pkg - (struct razor_package *) set->packages.data;
   494 			r = list_next(r);
   495 		}
   496 	}
   497 
   498 	prop_end = set->properties.data + set->properties.size;
   499 	a = pkgs;
   500 	for (prop = set->properties.data; prop < prop_end; prop++, a++) {
   501 		list_set_array(&prop->packages, &set->package_pool, a, 0);
   502 		array_release(a);
   503 	}
   504 	free(pkgs);
   505 }
   506 
   507 static void
   508 rebuild_file_package_lists(struct razor_set *set)
   509 {
   510 	struct array *pkgs, *a;
   511 	struct razor_package *pkg, *pkg_end;
   512 	struct razor_entry *entry, *entry_end;
   513 	struct list *r;
   514 	uint32_t *q;
   515 	int count;
   516 
   517 	count = set->files.size / sizeof (struct razor_entry);
   518 	pkgs = zalloc(count * sizeof *pkgs);
   519 	pkg_end = set->packages.data + set->packages.size;
   520 
   521 	for (pkg = set->packages.data; pkg < pkg_end; pkg++) {
   522 		r = list_first(&pkg->files, &set->file_pool);
   523 		while (r) {
   524 			q = array_add(&pkgs[r->data], sizeof *q);
   525 			*q = pkg - (struct razor_package *) set->packages.data;
   526 			r = list_next(r);
   527 		}
   528 	}
   529 
   530 	entry_end = set->files.data + set->files.size;
   531 	a = pkgs;
   532 	for (entry = set->files.data; entry < entry_end; entry++, a++) {
   533 		list_set_array(&entry->packages, &set->package_pool, a, 0);
   534 		array_release(a);
   535 	}
   536 	free(pkgs);
   537 }
   538 
   539 struct razor_set *
   540 razor_merger_commit(struct razor_merger *merger)
   541 {
   542 	struct razor_package *p, *pend;
   543 
   544 	/* As we built the package list, we filled out a bitvector of
   545 	 * the properties that are referenced by the packages in the
   546 	 * new set.  Now we do a parallel loop through the properties
   547 	 * and emit those marked in the bit vector to the new set.  In
   548 	 * the process, we update the bit vector to actually map from
   549 	 * indices in the old property list to indices in the new
   550 	 * property list for both sets. */
   551 
   552 	merge_properties(merger);
   553 	merge_files(merger);
   554 
   555 	/* Now we loop through the packages again and emit the
   556 	 * property lists, remapped to point to the new properties. */
   557 
   558 	pend = merger->set->packages.data + merger->set->packages.size;
   559 	for (p = merger->set->packages.data; p < pend; p++) {
   560 		struct source *src;
   561 
   562 		if (p->flags & UPSTREAM_SOURCE)
   563 			src = &merger->source2;
   564 		else
   565 			src = &merger->source1;
   566 
   567 		emit_properties(&p->properties,
   568 				&src->set->property_pool,
   569 				src->property_map,
   570 				&merger->set->property_pool);
   571 		emit_files(&p->files,
   572 			   &src->set->file_pool,
   573 			   src->file_map,
   574 			   &merger->set->file_pool);
   575 		p->flags &= ~UPSTREAM_SOURCE;
   576 	}
   577 
   578 	rebuild_property_package_lists(merger->set);
   579 	rebuild_file_package_lists(merger->set);
   580 
   581 	merger->committed = 1;
   582 
   583 	return merger->set;
   584 }
   585 
   586 void
   587 razor_merger_package_add_script(struct razor_merger *merger,
   588 				struct razor_package *package,
   589 				enum razor_property_flags script,
   590 				const char *program, const char *body)
   591 {
   592 	uint32_t p, b;
   593 	char *pool;
   594 	struct razor_set *set = merger->set;
   595 	assert ((void *)package >= set->packages.data && \
   596 	    (void *)package < set->packages.data + set->packages.size);
   597 
   598 	p = hashtable_tokenize(&merger->table, program);
   599 	b = hashtable_tokenize(&merger->table, body);
   600 
   601 	pool = merger->set->string_pool.data;
   602 
   603 	switch (script) {
   604 	case RAZOR_PROPERTY_PREUN:
   605 		if (package->preun.program != p) {
   606 			assert(pool[package->preun.program] == '\0');
   607 			package->preun.program = p;
   608 		}
   609 		if (package->preun.body != b) {
   610 			assert(pool[package->preun.body] == '\0');
   611 			package->preun.body = b;
   612 		}
   613 		break;
   614 	case RAZOR_PROPERTY_POSTUN:
   615 		if (package->postun.program != p) {
   616 			assert(pool[package->postun.program] == '\0');
   617 			package->postun.program = p;
   618 		}
   619 		if (package->postun.body != b) {
   620 			assert(pool[package->postun.body] == '\0');
   621 			package->postun.body = b;
   622 		}
   623 		break;
   624 	default:
   625 		break;
   626 	}
   627 }
   628 
   629 void razor_merger_destroy(struct razor_merger *merger)
   630 {
   631 	hashtable_release(&merger->table);
   632 	hashtable_release(&merger->file_table);
   633 	hashtable_release(&merger->details_table);
   634 	free(merger);
   635 }