librazor/importer.c
author J. Ali Harlow <ali@juiblex.co.uk>
Thu Oct 09 17:27:41 2014 +0100 (2014-10-09)
changeset 455 df914f383f5c
parent 438 fab0b8a61dcb
child 458 3f841a46eab5
permissions -rw-r--r--
Support downloading from local repository even without libcurl

Using the --url option of the razor executable, it is possible
to specify a yum repository on the local machine (eg., on installation
media) and import from there, eg.,:

C> razor --url file:///d:/ import-yum

This will be handled by libcurl if available but if not, an internal
copy routine will be used.

Note that if Microsoft's KTM implementation of atomic transactions is
used, then the current directory must support atomic transactions
(also improve error messages for this, and other, cases).
     1 /*
     2  * Copyright (C) 2008  Kristian Høgsberg <krh@redhat.com>
     3  * Copyright (C) 2008  Red Hat, Inc
     4  * Copyright (C) 2009  J. Ali Harlow <ali@juiblex.co.uk>
     5  *
     6  * This program is free software; you can redistribute it and/or modify
     7  * it under the terms of the GNU General Public License as published by
     8  * the Free Software Foundation; either version 2 of the License, or
     9  * (at your option) any later version.
    10  *
    11  * This program is distributed in the hope that it will be useful,
    12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
    13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    14  * GNU General Public License for more details.
    15  *
    16  * You should have received a copy of the GNU General Public License along
    17  * with this program; if not, write to the Free Software Foundation, Inc.,
    18  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
    19  */
    20 
    21 #define _GNU_SOURCE
    22 
    23 #include "config.h"
    24 #include <string.h>
    25 #include "razor-internal.h"
    26 #include "razor.h"
    27 
    28 /**
    29  * razor_importer_create:
    30  *
    31  * Create a new %razor_importer.
    32  *
    33  * Returns: the new %razor_importer.
    34  **/
    35 RAZOR_EXPORT struct razor_importer *
    36 razor_importer_create(void)
    37 {
    38 	struct razor_importer *importer;
    39 
    40 	importer = zalloc(sizeof *importer);
    41 	importer->set = razor_set_create();
    42 	hashtable_init(&importer->table, &importer->set->string_pool);
    43 	hashtable_init(&importer->details_table,
    44 		       &importer->set->details_string_pool);
    45 	hashtable_init(&importer->file_table,
    46 		       &importer->set->file_string_pool);
    47 
    48 	return importer;
    49 }
    50 
    51 /**
    52  * razor_importer_destroy:
    53  * @importer: the %razor_importer
    54  *
    55  * Destroy an importer without creating a %razor_set.  Normally,
    56  * %razor_importer_finish will create a new %razor_set and destroy the
    57  * importer.  If the import must be aborted without creating the set,
    58  * just destroy the import using this function.
    59  **/
    60 RAZOR_EXPORT void
    61 razor_importer_destroy(struct razor_importer *importer)
    62 {
    63 	/* FIXME: write this */
    64 }
    65 
    66 
    67 /**
    68  * razor_importer_begin_package:
    69  * @importer: the %razor_importer
    70  * @name: the name of the new package
    71  * @version: the version of the new package
    72  * @arch: the architechture of the new package.
    73  *
    74  * Begin describing a new package to the importer.  This creates a new
    75  * package and sets the %name, %version and %arch.  Subsequent calls
    76  * to %razor_importer_add_details, %razor_importer_add_property and
    77  * %razor_importer_add_file further describe this package and
    78  * %razor_importer_finish_package marks the end of meta data for this
    79  * package.
    80  **/
    81 RAZOR_EXPORT void
    82 razor_importer_begin_package(struct razor_importer *importer,
    83 			     const char *name,
    84 			     const char *version,
    85 			     const char *arch)
    86 {
    87 	uint32_t empty;
    88 	struct razor_package *p;
    89 
    90 	p = array_add(&importer->set->packages, sizeof *p);
    91 	p->name = hashtable_tokenize(&importer->table, name);
    92 	p->flags = 0;
    93 	p->version = hashtable_tokenize(&importer->table, version);
    94 	p->arch = hashtable_tokenize(&importer->table, arch);
    95 
    96 	importer->package = p;
    97 	array_init(&importer->properties);
    98 	array_init(&importer->install_prefixes);
    99 
   100 	empty = hashtable_tokenize(&importer->details_table, "");
   101 	importer->package->preun.program = empty;
   102 	importer->package->preun.body = empty;
   103 	importer->package->postun.program = empty;
   104 	importer->package->postun.body = empty;
   105 }
   106 
   107 /**
   108  * razor_importer_finish_package:
   109  * @importer: the %razor_importer
   110  *
   111  * Tells the importer that the current package is complete.
   112  **/
   113 RAZOR_EXPORT void
   114 razor_importer_finish_package(struct razor_importer *importer)
   115 {
   116 	list_set_array(&importer->package->properties,
   117 		       &importer->set->property_pool,
   118 		       &importer->properties,
   119 		       1);
   120 
   121 	list_set_array(&importer->package->install_prefixes,
   122 		       &importer->set->prefix_pool,
   123 		       &importer->install_prefixes,
   124 		       0);
   125 
   126 	array_release(&importer->properties);
   127 	array_release(&importer->install_prefixes);
   128 }
   129 
   130 /**
   131  * razor_importer_add_details:
   132  * @importer: the %razor_importer
   133  * @summary: the package summary
   134  * @description: the package description
   135  * @url: the package url
   136  * @license: the package license
   137  *
   138  * Provide additional information for the current package.
   139  **/
   140 RAZOR_EXPORT void
   141 razor_importer_add_details(struct razor_importer *importer,
   142 			   const char *summary,
   143 			   const char *description,
   144 			   const char *url,
   145 			   const char *license)
   146 {
   147 	importer->package->summary = hashtable_tokenize(&importer->details_table, summary);
   148 	importer->package->description = hashtable_tokenize(&importer->details_table, description);
   149 	importer->package->url = hashtable_tokenize(&importer->details_table, url);
   150 	importer->package->license = hashtable_tokenize(&importer->details_table, license);
   151 }
   152 
   153 /**
   154  * razor_importer_add_script:
   155  * @importer: the %razor_importer
   156  * @script: either %RAZOR_PROPERTY_PREUN or %RAZOR_PROPERTY_POSTUN
   157  * @program: the program to run the script
   158  * @body: the body of the script
   159  *
   160  * Provide a script to use when uninstalling the current package.
   161  **/
   162 RAZOR_EXPORT void
   163 razor_importer_add_script(struct razor_importer *importer,
   164 			  enum razor_property_flags script,
   165 			  const char *program,
   166 			  const char *body)
   167 {
   168 	switch (script) {
   169 	case RAZOR_PROPERTY_PREUN:
   170 		importer->package->preun.program =
   171 			hashtable_tokenize(&importer->table, program);
   172 		importer->package->preun.body =
   173 			hashtable_tokenize(&importer->table, body);
   174 		break;
   175 	case RAZOR_PROPERTY_POSTUN:
   176 		importer->package->postun.program =
   177 			hashtable_tokenize(&importer->table, program);
   178 		importer->package->postun.body =
   179 			hashtable_tokenize(&importer->table, body);
   180 		break;
   181 	default:
   182 		break;
   183 	}
   184 }
   185 
   186 /**
   187  * razor_importer_add_install_prefixes:
   188  * @importer: the %razor_importer
   189  * @install_prefix: the relocated prefix
   190  *
   191  * Adds a relocated prefix for the current package.
   192  **/
   193 RAZOR_EXPORT void
   194 razor_importer_add_install_prefix(struct razor_importer *importer,
   195 				  const char *install_prefix)
   196 {
   197 	uint32_t *r;
   198 
   199 	r = array_add(&importer->install_prefixes, sizeof *r);
   200 	*r = hashtable_tokenize(&importer->table, install_prefix);
   201 }
   202 
   203 /**
   204  * razor_importer_add_property:
   205  * @importer: the %razor_importer
   206  * @name: name of the property
   207  * @flags: property flags
   208  * @version: version of the property or %NULL
   209  *
   210  * Add a property for the current package.  The %flags parameter
   211  * determines the type of the property and optionally the relation to
   212  * the specified version and the availability constraint.  See
   213  * %razor_property_flags for further information about the flag
   214  * parameter.
   215  **/
   216 RAZOR_EXPORT void
   217 razor_importer_add_property(struct razor_importer *importer,
   218 			    const char *name,
   219 			    uint32_t flags,
   220 			    const char *version)
   221 {
   222 	struct razor_property *p;
   223 	uint32_t *r;
   224 
   225 	p = array_add(&importer->set->properties, sizeof *p);
   226 	p->name = hashtable_tokenize(&importer->table, name);
   227 	p->flags = flags;
   228 	p->version = hashtable_tokenize(&importer->table, version);
   229 	list_set_ptr(&p->packages, importer->package -
   230 		     (struct razor_package *) importer->set->packages.data);
   231 
   232 	r = array_add(&importer->properties, sizeof *r);
   233 	*r = p - (struct razor_property *) importer->set->properties.data;
   234 
   235 	if (((flags & RAZOR_PROPERTY_TYPE_MASK) == RAZOR_PROPERTY_REQUIRES) &&
   236 	    *name == '/') {
   237 		r = array_add(&importer->file_requires, sizeof *r);
   238 		*r = p->name;
   239 	}
   240 }
   241 
   242 /**
   243  * razor_importer_add_file:
   244  * @importer: the %razor_importer
   245  * @name: name of the file
   246  *
   247  * Add a file to the current package.
   248  **/
   249 RAZOR_EXPORT void
   250 razor_importer_add_file(struct razor_importer *importer, const char *name)
   251 {
   252 	struct import_entry *e;
   253 
   254 	e = array_add(&importer->files, sizeof *e);
   255 
   256 	e->package = importer->package -
   257 		(struct razor_package *) importer->set->packages.data;
   258 	e->name = strdup(name);
   259 }
   260 
   261 static int
   262 compare_packages(const void *p1, const void *p2, void *data)
   263 {
   264 	const struct razor_package *pkg1 = p1, *pkg2 = p2;
   265 	struct razor_set *set = data;
   266 	char *pool = set->string_pool.data;
   267 
   268 	/* FIXME: what if the flags are different? */
   269 	if (pkg1->name == pkg2->name)
   270 		return razor_versioncmp(&pool[pkg1->version], &pool[pkg2->version]);
   271 	else
   272 		return strcmp(&pool[pkg1->name], &pool[pkg2->name]);
   273 }
   274 
   275 static int
   276 compare_properties(const void *p1, const void *p2, void *data)
   277 {
   278 	const struct razor_property *prop1 = p1, *prop2 = p2;
   279 	struct razor_set *set = data;
   280 	char *pool = set->string_pool.data;
   281 
   282 	if (prop1->name != prop2->name)
   283 		return strcmp(&pool[prop1->name], &pool[prop2->name]);
   284 	else if (prop1->flags != prop2->flags)
   285 		return prop1->flags - prop2->flags;
   286 	else if (prop1->version != prop2->version)
   287 		return razor_versioncmp(&pool[prop1->version], &pool[prop2->version]);
   288 	else
   289 		return prop1->packages.list_ptr - prop2->packages.list_ptr;
   290 }
   291 
   292 static uint32_t *
   293 uniqueify_properties(struct razor_set *set)
   294 {
   295 	struct razor_property *rp, *up, *rp_end;
   296 	struct array *pkgs, *p;
   297 	struct list_head *r;
   298 	uint32_t *map, *rmap;
   299 	int i, count, unique;
   300 
   301 	count = set->properties.size / sizeof(struct razor_property);
   302 	map = razor_qsort_with_data(set->properties.data,
   303 				    count,
   304 				    sizeof(struct razor_property),
   305 				    compare_properties,
   306 				    set);
   307 
   308 	rp_end = set->properties.data + set->properties.size;
   309 	rmap = malloc(count * sizeof *map);
   310 	pkgs = zalloc(count * sizeof *pkgs);
   311 	for (rp = set->properties.data, up = rp, i = 0; rp < rp_end; rp++, i++) {
   312 		if (rp->name != up->name ||
   313 		    rp->flags != up->flags ||
   314 		    rp->version != up->version) {
   315 			up++;
   316 			up->name = rp->name;
   317 			up->flags = rp->flags;
   318 			up->version = rp->version;
   319 		}
   320 
   321 		unique = up - (struct razor_property *) set->properties.data;
   322 		rmap[map[i]] = unique;
   323 		r = array_add(&pkgs[unique], sizeof *r);
   324 		*r = rp->packages;
   325 	}
   326 	free(map);
   327 
   328 	if (up != rp)
   329 		up++;
   330 	set->properties.size = (void *) up - set->properties.data;
   331 	rp_end = up;
   332 	for (rp = set->properties.data, p = pkgs; rp < rp_end; rp++, p++) {
   333 		list_set_array(&rp->packages, &set->package_pool, p, 0);
   334 		array_release(p);
   335 	}
   336 
   337 	free(pkgs);
   338 
   339 	return rmap;
   340 }
   341 
   342 static int
   343 compare_filenames(const void *p1, const void *p2, void *data)
   344 {
   345 	const struct import_entry *e1 = p1;
   346 	const struct import_entry *e2 = p2;
   347 	const char *n1 = e1->name;
   348 	const char *n2 = e2->name;
   349 
   350 	/* Need to make sure that the contents of a directory
   351 	 * are sorted immediately after it. So "foo/bar" has to
   352 	 * sort before "foo.conf"
   353 	 *
   354 	 * FIXME: this is about 60% slower than strcmp
   355 	 */
   356 	while (*n1 && *n2) {
   357 		if (*n1 < *n2)
   358 			return *n2 == '/' ? 1 : -1;
   359 		else if (*n1 > *n2)
   360 			return *n1 == '/' ? -1 : 1;
   361 		n1++;
   362 		n2++;
   363 	}
   364 	if (*n1)
   365 		return 1;
   366 	else if (*n2)
   367 		return -1;
   368 	else
   369 		return 0;
   370 }
   371 
   372 static void
   373 count_entries(struct import_directory *d)
   374 {
   375 	struct import_directory *p, *end;
   376 
   377 	p = d->files.data;
   378 	end = d->files.data + d->files.size;
   379 	d->count = 0;
   380 	while (p < end) {
   381 		count_entries(p);
   382 		d->count += p->count + 1;
   383 		p++;
   384 	}
   385 }
   386 
   387 static void
   388 serialize_files(struct razor_set *set,
   389 		struct import_directory *d, struct array *array)
   390 {
   391 	struct import_directory *p, *end;
   392 	struct razor_entry *e = NULL;
   393 	uint32_t s;
   394 
   395 	p = d->files.data;
   396 	end = d->files.data + d->files.size;
   397 	s = array->size / sizeof *e + d->files.size / sizeof *p;
   398 	while (p < end) {
   399 		e = array_add(array, sizeof *e);
   400 		e->name = p->name;
   401 		e->flags = 0;
   402 		e->start = p->count > 0 ? s : 0;
   403 		s += p->count;
   404 
   405 		list_set_array(&e->packages, &set->package_pool, &p->packages, 0);
   406 		array_release(&p->packages);
   407 		p++;
   408 	}
   409 	if (e != NULL)
   410 		e->flags |= RAZOR_ENTRY_LAST;
   411 
   412 	p = d->files.data;
   413 	end = d->files.data + d->files.size;
   414 	while (p < end) {
   415 		serialize_files(set, p, array);
   416 		p++;
   417 	}
   418 }
   419 
   420 static void
   421 remap_property_package_links(struct array *properties, uint32_t *rmap)
   422 {
   423 	struct razor_property *p, *end;
   424 
   425 	end = properties->data + properties->size;
   426 	for (p = properties->data; p < end; p++)
   427 		list_remap_head(&p->packages, rmap);
   428 }
   429 
   430 static void
   431 build_file_tree(struct razor_importer *importer)
   432 {
   433 	int count, i, length;
   434 	struct import_entry *filenames;
   435 	char *f, *end;
   436 	uint32_t name, *r, s;
   437 	char dirname[256];
   438 	struct import_directory *d, *last_root;
   439 	struct array roots;
   440 	struct razor_entry *e;
   441 
   442 	count = importer->files.size / sizeof (struct import_entry);
   443 	filenames = importer->files.data;
   444 	razor_qsort_with_data(filenames,
   445 			      count,
   446 			      sizeof (struct import_entry),
   447 			      compare_filenames,
   448 			      NULL);
   449 
   450 	array_init(&roots);
   451 	last_root = NULL;
   452 
   453 	for (i = 0; i < count; i++) {
   454 		f = filenames[i].name;
   455 		d = NULL;
   456 		while (*f) {
   457 			end = strchr(f, '/');
   458 			if (end == NULL)
   459 				end = f + strlen(f);
   460 			length = end - f;
   461 			memcpy(dirname, f, length);
   462 			dirname[length] = '\0';
   463 			name = hashtable_tokenize(&importer->file_table,
   464 						  dirname);
   465 			if (!d) {
   466 				if (!last_root || last_root->name != name) {
   467 					d = array_add(&roots, sizeof *d);
   468 					d->name = name;
   469 					d->last = NULL;
   470 					array_init(&d->files);
   471 					array_init(&d->packages);
   472 					last_root = d;
   473 				}
   474 				d = last_root;
   475 			} else {
   476 				if (!d->last || d->last->name != name) {
   477 					d->last = array_add(&d->files,
   478 							    sizeof *d);
   479 					d->last->name = name;
   480 					d->last->last = NULL;
   481 					array_init(&d->last->files);
   482 					array_init(&d->last->packages);
   483 				}
   484 				d = d->last;
   485 			}
   486 			f = end + 1;
   487 			if (*end == '\0')
   488 				break;
   489 		}
   490 
   491 		r = array_add(&d->packages, sizeof *r);
   492 		*r = filenames[i].package;
   493 		free(filenames[i].name);
   494 	}
   495 
   496 	count = roots.size / sizeof (struct import_directory);
   497 	d = roots.data;
   498 	s = count;
   499 	for (i = 0; i < count; i++) {
   500 		count_entries(d);
   501 		if (i)
   502 			e = array_add(&importer->set->files, sizeof *e);
   503 		else
   504 			e = importer->set->files.data;
   505 		e->name = d->name;
   506 		e->flags = 0;
   507 		e->start = d->count > 0 ? s : 0;
   508 		s += d->count;
   509 		list_set_empty(&e->packages);
   510 		d++;
   511 	}
   512 	if (count)
   513 		e->flags |= RAZOR_ENTRY_LAST;
   514 
   515 	d = roots.data;
   516 	for (i = 0; i < count; i++) {
   517 		serialize_files(importer->set, d, &importer->set->files);
   518 		d++;
   519 	}
   520 
   521 	array_release(&importer->files);
   522 	array_release(&roots);
   523 }
   524 
   525 static void
   526 list_to_array(struct list *list, struct array *array)
   527 {
   528 	uint32_t *item;
   529 
   530 	while (list) {
   531 		 item = array_add(array, sizeof *item);
   532 		 *item = list->data;
   533 		 list = list_next(list);
   534 	}
   535 }
   536 
   537 static int
   538 compare_file_requires(const void *p1, const void *p2, void *data)
   539 {
   540 	uint32_t *f1 = (void *)p1, *f2 = (void *)p2;
   541 	const char *pool = data;
   542 
   543 	return strcmp(&pool[*f1], &pool[*f2]);
   544 }
   545 
   546 static void
   547 find_file_provides(struct razor_importer *importer)
   548 {
   549 	struct razor_property *prop;
   550 	struct razor_entry *top, *entry;
   551 	struct razor_package *packages;
   552 	struct array pkgprops;
   553 	struct list *pkg;
   554 	uint32_t *req, *req_start, *req_end;
   555 	uint32_t *map, *newprop;
   556 	char *pool;
   557 
   558 	pool = importer->set->string_pool.data;
   559 	packages = importer->set->packages.data;
   560 	top = importer->set->files.data;
   561 
   562 	req = req_start = importer->file_requires.data;
   563 	req_end = importer->file_requires.data + importer->file_requires.size;
   564 	map = razor_qsort_with_data(req, req_end - req, sizeof *req,
   565 				    compare_file_requires, pool);
   566 	free(map);
   567 
   568 	for (req = req_start; req < req_end; req++) {
   569 		if (req > req_start && req[0] == req[-1])
   570 			continue;
   571 		entry = razor_set_find_entry(importer->set, top, &pool[*req]);
   572 		if (!entry)
   573 			continue;
   574 
   575 		for (pkg = list_first(&entry->packages, &importer->set->package_pool); pkg; pkg = list_next(pkg)) {
   576 			prop = array_add(&importer->set->properties, sizeof *prop);
   577 			prop->name = *req;
   578 			prop->flags =
   579 				RAZOR_PROPERTY_PROVIDES | RAZOR_PROPERTY_EQUAL;
   580 			prop->version = hashtable_tokenize(&importer->table, "");
   581 			list_set_ptr(&prop->packages, pkg->data);
   582 
   583 			/* Update property list of pkg */
   584 			array_init(&pkgprops);
   585 			list_to_array(list_first(&packages[pkg->data].properties, &importer->set->property_pool), &pkgprops);
   586 			newprop = array_add(&pkgprops, sizeof *newprop);
   587 			*newprop = prop - (struct razor_property *)importer->set->properties.data;
   588 			list_set_array(&packages[pkg->data].properties, &importer->set->property_pool, &pkgprops, 1);
   589 			array_release(&pkgprops);
   590 		}
   591 	}
   592 
   593 	array_release(&importer->file_requires);
   594 }
   595 
   596 static void
   597 build_package_file_lists(struct razor_set *set, uint32_t *rmap)
   598 {
   599 	struct razor_package *p, *packages;
   600 	struct array *pkgs;
   601 	struct razor_entry *e, *end;
   602 	struct list *r;
   603 	uint32_t *q;
   604 	int i, count;
   605 
   606 	count = set->packages.size / sizeof *p;
   607 	pkgs = zalloc(count * sizeof *pkgs);
   608 
   609 	end = set->files.data + set->files.size;
   610 	for (e = set->files.data; e < end; e++) {
   611 		list_remap_head(&e->packages, rmap);
   612 		r = list_first(&e->packages, &set->package_pool);
   613 		while (r) {
   614 			q = array_add(&pkgs[r->data], sizeof *q);
   615 			*q = e - (struct razor_entry *) set->files.data;
   616 			r = list_next(r);
   617 		}
   618 	}
   619 
   620 	packages = set->packages.data;
   621 	for (i = 0; i < count; i++) {
   622 		list_set_array(&packages[i].files, &set->file_pool, &pkgs[i], 0);
   623 		array_release(&pkgs[i]);
   624 	}
   625 	free(pkgs);
   626 }
   627 
   628 /**
   629  * razor_importer_finish:
   630  * @importer: the %razor_importer
   631  *
   632  * Finish importing packages and create the package set.  This sorts
   633  * and indexes all the packages, properties and files in the importer
   634  * and creates a new %razor_set.  After creating the new package set,
   635  * the importer is destroyed.
   636  *
   637  * Returns: the new %razor_set
   638  **/
   639 RAZOR_EXPORT struct razor_set *
   640 razor_importer_finish(struct razor_importer *importer)
   641 {
   642 	struct razor_set *set;
   643 	uint32_t *map, *rmap;
   644 	int i, count;
   645 
   646 	build_file_tree(importer);
   647 	find_file_provides(importer);
   648 
   649 	map = uniqueify_properties(importer->set);
   650 	list_remap_pool(&importer->set->property_pool, map);
   651 	free(map);
   652 
   653 	count = importer->set->packages.size / sizeof(struct razor_package);
   654 	map = razor_qsort_with_data(importer->set->packages.data,
   655 				    count,
   656 				    sizeof(struct razor_package),
   657 				    compare_packages,
   658 				    importer->set);
   659 
   660 	rmap = malloc(count * sizeof *rmap);
   661 	for (i = 0; i < count; i++)
   662 		rmap[map[i]] = i;
   663 	free(map);
   664 
   665 	list_remap_pool(&importer->set->package_pool, rmap);
   666 	build_package_file_lists(importer->set, rmap);
   667 	remap_property_package_links(&importer->set->properties, rmap);
   668 	free(rmap);
   669 
   670 	set = importer->set;
   671 	hashtable_release(&importer->table);
   672 	hashtable_release(&importer->details_table);
   673 	hashtable_release(&importer->file_table);
   674 	free(importer);
   675 
   676 	return set;
   677 }