# HG changeset patch # User J. Ali Harlow # Date 1467633859 -3600 # Node ID 48e45439fd9ac8f4c33d6d1faa87237426222a34 # Parent 008c75a5e08d8194c79b46d33811cd2d8b003de4 Support getting file contents from archives diff -r 008c75a5e08d -r 48e45439fd9a configure.ac --- a/configure.ac Mon Jul 04 10:48:18 2016 +0100 +++ b/configure.ac Mon Jul 04 13:04:19 2016 +0100 @@ -205,6 +205,25 @@ AC_MSG_RESULT($have_gcc4) REQUIREMENTS="" +AC_ARG_WITH([libarchive], + [AS_HELP_STRING([--without-libarchive], + [disable support for libarchive])], + [], + [with_libarchive=yes]) + +LIBARCHIVE_CFLAGS= +LIBARCHIVE_LIBS= +AS_IF([test "x$with_libarchive" != xno], + [PKG_CHECK_MODULES(LIBARCHIVE, [libarchive]) + REQUIREMENTS="$REQUIREMENTS libarchive" + AC_DEFINE([HAVE_LIBARCHIVE], [1], [Define if you have libarchive])]) +AC_SUBST(LIBARCHIVE_CFLAGS) +AC_SUBST(LIBARCHIVE_LIBS) + +AS_IF([test "x$with_libarchive" != xno], + [AC_PATH_PROG(ZIP, [zip], [])]) +AM_CONDITIONAL([HAVE_ZIP], [test -n "$ZIP"]) + AC_ARG_WITH([curl], [AS_HELP_STRING([--without-curl], [disable support for curl])], [], @@ -217,6 +236,7 @@ AC_DEFINE([HAVE_CURL], [1], [Define if you have curl])]) AC_SUBST(CURL_CFLAGS) AC_SUBST(CURL_LIBS) +AM_CONDITIONAL([HAVE_CURL], [test "x$with_curl" != xno]) ZLIB_LIBS="" AC_ARG_WITH(zlib, [ --with-zlib= Use zlib from here], diff -r 008c75a5e08d -r 48e45439fd9a librazor/Makefile.am --- a/librazor/Makefile.am Mon Jul 04 10:48:18 2016 +0100 +++ b/librazor/Makefile.am Mon Jul 04 13:04:19 2016 +0100 @@ -55,8 +55,8 @@ librazor_la_SOURCES += lua.c endif -librazor_la_LIBADD = $(ZLIB_LIBS) types/libtypes.la $(LUA_LIBS) \ - ../gl/libgnu.la $(INTLLIBS) $(EXTRA_LIBS) +librazor_la_LIBADD = $(LIBARCHIVE_LIBS) $(ZLIB_LIBS) types/libtypes.la \ + $(LUA_LIBS) ../gl/libgnu.la $(INTLLIBS) $(EXTRA_LIBS) librazor_la_LDFLAGS = -no-undefined -export-symbols-regex '^razor_' \ -version-info $(LT_CURRENT):$(LT_REVISION):$(LT_AGE) @@ -76,7 +76,8 @@ if HAVE_LUA test_lua_SOURCES = test-lua.c test_lua_LDADD = lua.lo uri.lo uri-io.lo util.lo path.lo error.lo \ - types/libtypes.la $(LUA_LIBS) ../gl/libgnu.la $(INTLLIBS) $(EXTRA_LIBS) + types/libtypes.la $(LIBARCHIVE_LIBS) $(LUA_LIBS) ../gl/libgnu.la \ + $(INTLLIBS) $(EXTRA_LIBS) TESTS += test-lua endif diff -r 008c75a5e08d -r 48e45439fd9a librazor/razor-internal.h --- a/librazor/razor-internal.h Mon Jul 04 10:48:18 2016 +0100 +++ b/librazor/razor-internal.h Mon Jul 04 13:04:19 2016 +0100 @@ -257,22 +257,6 @@ char *razor_file_readdir(void *dir, struct razor_error **error); int razor_file_closedir(void *dir, struct razor_error **error); -int razor_uri_mkdir(const char *uri, mode_t mode, struct razor_error **error); -int razor_uri_unlink(const char *uri, struct razor_error **error); -int razor_uri_open(const char *uri, int flags, mode_t mode, - struct razor_error **error); -int razor_uri_move(const char *uri, const char *dest, - struct razor_error **error); -int razor_uri_is_directory(const char *uri, struct razor_error **error); -char *razor_uri_mkdtemp_near(const char *uri, const char *template, - struct razor_error **error) RAZOR_MALLOC; -void *razor_uri_opendir(const char *uri, struct razor_error **error); -char *razor_uri_readdir(void *dir, struct razor_error **error) RAZOR_MALLOC; -int razor_uri_closedir(void *dir, struct razor_error **error); -void *razor_uri_get_contents(const char *uri, size_t *length, int private, - struct razor_error **error); -int razor_uri_free_contents(void *addr, size_t length); - char *razor_path_from_parsed_uri(const struct razor_uri *ru, struct razor_error **error); diff -r 008c75a5e08d -r 48e45439fd9a librazor/razor.h.in --- a/librazor/razor.h.in Mon Jul 04 10:48:18 2016 +0100 +++ b/librazor/razor.h.in Mon Jul 04 13:04:19 2016 +0100 @@ -592,9 +592,6 @@ struct razor_set *razor_importer_finish(struct razor_importer *importer); -struct razor_set *razor_set_create_from_yum(void); -struct razor_set *razor_set_create_from_rpmdb(void); - /** * SECTION:root * @title: Root @@ -650,6 +647,22 @@ int (*closedir)(void *dir, struct razor_error **error); }; +int razor_uri_mkdir(const char *uri, mode_t mode, struct razor_error **error); +int razor_uri_unlink(const char *uri, struct razor_error **error); +int razor_uri_open(const char *uri, int flags, mode_t mode, + struct razor_error **error); +int razor_uri_move(const char *uri, const char *dest, + struct razor_error **error); +int razor_uri_is_directory(const char *uri, struct razor_error **error); +char *razor_uri_mkdtemp_near(const char *uri, const char *_template, + struct razor_error **error) RAZOR_MALLOC; +void *razor_uri_opendir(const char *uri, struct razor_error **error); +char *razor_uri_readdir(void *dir, struct razor_error **error) RAZOR_MALLOC; +int razor_uri_closedir(void *dir, struct razor_error **error); +void *razor_uri_get_contents(const char *uri, size_t *length, int _private, + struct razor_error **error); +int razor_uri_free_contents(void *addr, size_t length); + int razor_uri_set_vtable(const char *scheme, struct razor_uri_vtable *vtable, struct razor_error **error); @@ -680,7 +693,7 @@ char *razor_path_from_uri(const char *uri, struct razor_error **error) RAZOR_MALLOC; char *razor_path_to_uri(const char *path) RAZOR_MALLOC; -char *razor_path_relative_to_uri(const char *file_uri, const char *path, +char *razor_path_relative_to_uri(const char *uri, const char *path, struct razor_error **error) RAZOR_MALLOC; const char *razor_system_arch(void); diff -r 008c75a5e08d -r 48e45439fd9a librazor/uri-io.c --- a/librazor/uri-io.c Mon Jul 04 10:48:18 2016 +0100 +++ b/librazor/uri-io.c Mon Jul 04 13:04:19 2016 +0100 @@ -38,6 +38,10 @@ #if HAVE_SYS_MMAN_H #include #endif +#if HAVE_LIBARCHIVE +#include +#include +#endif #include #include "razor.h" @@ -51,6 +55,130 @@ #define strcmp0(s1, s2) ((s1) && (s2) ? strcmp(s1, s2) : \ (s1) ? 1 : (s2) ? -1 : 0) +#if HAVE_LIBARCHIVE +static void * +razor_archive_get_file_contents(const char *filename, int fd, const char *path, + size_t *length, struct razor_error **error) +{ + int r; + void *addr; + const void *buf; + size_t size; + off_t offset; + struct archive *a; + struct archive_entry *entry; + + a = archive_read_new(); + archive_read_support_compression_all(a); + archive_read_support_format_all(a); + + r = archive_read_open_fd(a, fd, 10240); + + if (r) { + razor_set_error(error, RAZOR_POSIX_ERROR, archive_errno(a), + filename, archive_error_string(a)); + archive_read_finish(a); + return NULL; + } + + for (;;) { + r = archive_read_next_header(a, &entry); + if (r == ARCHIVE_EOF) + break; + else if (r < ARCHIVE_WARN) { + razor_set_error(error, RAZOR_POSIX_ERROR, + archive_errno(a), filename, + archive_error_string(a)); + archive_read_close(a); + archive_read_finish(a); + return NULL; + } + + /* + * TODO: Unicode support. Might need to wait for libarchive v4. + */ + if (!strcmp(archive_entry_pathname(entry), path)) { + addr = malloc(archive_entry_size(entry)); + if (!addr) { + archive_read_close(a); + archive_read_finish(a); + razor_set_error(error, RAZOR_POSIX_ERROR, + ENOMEM, NULL, + "Not enough memory"); + return NULL; + } + + for(;;) { + r = archive_read_data_block(a, &buf, &size, + &offset); + if (r == ARCHIVE_EOF) + break; + if (r < ARCHIVE_OK) { + razor_set_error(error, RAZOR_POSIX_ERROR, + archive_errno(a), path, + archive_error_string(a)); + free(addr); + addr = NULL; + break; + } + memcpy((char *)addr + offset, buf, size); + } + + archive_read_close(a); + archive_read_finish(a); + + return addr; + } + } + + archive_read_close(a); + archive_read_finish(a); + + razor_set_error(error, RAZOR_POSIX_ERROR, ENOENT, path, + "No such file or directory in archive"); + + return NULL; +} + +static void * +razor_file_get_contents_archive(const char *filename, size_t *length, + struct razor_error **error) +{ + int fd; + char *path, *slash, *s; + void *addr; + + path = strdup(filename); + slash = strrchr(path, '/'); + + while (slash) { + *slash = '\0'; + fd = open(path, O_RDONLY | O_BINARY); + if (fd >= 0) { + addr = razor_archive_get_file_contents(path, fd, + slash + 1, + length, error); + free(path); + close(fd); + return addr; + } else if (errno != ENOTDIR) { + free(path); + razor_set_error_posix(error, filename); + return NULL; + } + s = strrchr(path, '/'); + *slash = '/'; + slash = s; + } + + /* Impossible */ + free(path); + razor_set_error(error, RAZOR_POSIX_ERROR, ENOTDIR, filename, + strerror(ENOTDIR)); + return NULL; +} +#endif + #define OPEN_FILE_USED (1U<<0) #define OPEN_FILE_MMAPPED (1U<<1) @@ -68,24 +196,33 @@ int fd; struct stat st; void *addr = NULL; - size_t nb; + size_t nb, size; ssize_t res; struct open_file *of, *ofend; fd = open(filename, O_RDONLY | O_BINARY); if (fd < 0) { +#if HAVE_LIBARCHIVE + if (errno != ENOTDIR) { + razor_set_error_posix(error, filename); + return NULL; + } + addr = razor_file_get_contents_archive(filename, &size, error); + if (!addr) + return NULL; +#else razor_set_error_posix(error, filename); return NULL; +#endif + } else { + if (fstat(fd, &st) < 0) { + razor_set_error_posix(error, filename); + close(fd); + return NULL; + } + size = st.st_size; } - if (fstat(fd, &st) < 0) { - razor_set_error_posix(error, filename); - close(fd); - return NULL; - } - - *length = st.st_size; - ofend = open_files.data + open_files.size; for (of = open_files.data; of < ofend; of++) if (!(of->flags & OPEN_FILE_USED)) @@ -96,21 +233,21 @@ } #if HAVE_SYS_MMAN_H - if (!private) { - addr = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0); + if (!addr && !private) { + addr = mmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0); if (addr == MAP_FAILED) addr = NULL; else of->flags = OPEN_FILE_USED | OPEN_FILE_MMAPPED; } #endif /* HAVE_SYS_MMAN_H */ + if (!addr) { - addr = malloc(st.st_size); + addr = malloc(size); if (addr) { - of->flags = OPEN_FILE_USED; nb = 0; - while(nb < st.st_size) { - res = read(fd, addr + nb, st.st_size - nb); + while(nb < size) { + res = read(fd, addr + nb, size - nb); if (res <= 0) { razor_set_error_posix(error, filename); free(addr); @@ -119,14 +256,21 @@ } nb += res; } + if (addr) + of->flags = OPEN_FILE_USED; } else razor_set_error(error, RAZOR_POSIX_ERROR, ENOMEM, NULL, "Not enough memory"); } - close(fd); + + if (fd >= 0) + close(fd); of->addr = addr; - of->length = st.st_size; + of->length = size; + + if (addr) + *length = size; return addr; } @@ -659,7 +803,8 @@ return fallback; } -int razor_uri_mkdir(const char *uri, mode_t mode, struct razor_error **error) +RAZOR_EXPORT int +razor_uri_mkdir(const char *uri, mode_t mode, struct razor_error **error) { int retval; char *path; @@ -700,7 +845,8 @@ return retval; } -int razor_uri_unlink(const char *uri, struct razor_error **error) +RAZOR_EXPORT int +razor_uri_unlink(const char *uri, struct razor_error **error) { int retval; char *path; @@ -741,8 +887,9 @@ return retval; } -int razor_uri_open(const char *uri, int flags, mode_t mode, - struct razor_error **error) +RAZOR_EXPORT int +razor_uri_open(const char *uri, int flags, mode_t mode, + struct razor_error **error) { int retval; char *path; @@ -783,8 +930,9 @@ return retval; } -int razor_uri_move(const char *src_uri, const char *dst_uri, - struct razor_error **error) +RAZOR_EXPORT int +razor_uri_move(const char *src_uri, const char *dst_uri, + struct razor_error **error) { int retval; char *src_path, *dst_path; @@ -851,8 +999,9 @@ return retval; } -void *razor_uri_get_contents(const char *uri, size_t *length, int private, - struct razor_error **error) +RAZOR_EXPORT void * +razor_uri_get_contents(const char *uri, size_t *length, int private, + struct razor_error **error) { void *retval; char *path; @@ -897,7 +1046,7 @@ return retval; } -int razor_uri_free_contents(void *addr, size_t length) +RAZOR_EXPORT int razor_uri_free_contents(void *addr, size_t length) { int retval, of; struct razor_uri_vtable_entry *entry, *eend; @@ -925,7 +1074,8 @@ return retval; } -int razor_uri_is_directory(const char *uri, struct razor_error **error) +RAZOR_EXPORT int +razor_uri_is_directory(const char *uri, struct razor_error **error) { int retval; char *path; @@ -966,8 +1116,9 @@ return retval; } -char *razor_uri_mkdtemp_near(const char *uri, const char *template, - struct razor_error **error) +RAZOR_EXPORT char * +razor_uri_mkdtemp_near(const char *uri, const char *template, + struct razor_error **error) { char *retval, *s; char *path; @@ -1011,7 +1162,8 @@ return retval; } -void *razor_uri_opendir(const char *uri, struct razor_error **error) +RAZOR_EXPORT void * +razor_uri_opendir(const char *uri, struct razor_error **error) { void *retval; char *path; @@ -1055,7 +1207,7 @@ return retval; } -char *razor_uri_readdir(void *dir, struct razor_error **error) +RAZOR_EXPORT char *razor_uri_readdir(void *dir, struct razor_error **error) { int od; char *retval; @@ -1086,7 +1238,7 @@ return retval; } -int razor_uri_closedir(void *dir, struct razor_error **error) +RAZOR_EXPORT int razor_uri_closedir(void *dir, struct razor_error **error) { int od; int retval; diff -r 008c75a5e08d -r 48e45439fd9a src/Makefile.am --- a/src/Makefile.am Mon Jul 04 10:48:18 2016 +0100 +++ b/src/Makefile.am Mon Jul 04 13:04:19 2016 +0100 @@ -24,7 +24,7 @@ pkgdata_DATA = test.xml endif -razor_SOURCES = main.c import-yum.c +razor_SOURCES = main.c import.h import-yum.c if HAVE_RPMLIB razor_SOURCES += import-rpmdb.c endif diff -r 008c75a5e08d -r 48e45439fd9a src/import-rpmdb.c --- a/src/import-rpmdb.c Mon Jul 04 10:48:18 2016 +0100 +++ b/src/import-rpmdb.c Mon Jul 04 13:04:19 2016 +0100 @@ -32,6 +32,7 @@ #include #include "razor.h" +#include "import.h" union rpm_entry { void *p; diff -r 008c75a5e08d -r 48e45439fd9a src/import-yum.c --- a/src/import-yum.c Mon Jul 04 10:48:18 2016 +0100 +++ b/src/import-yum.c Mon Jul 04 13:04:19 2016 +0100 @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -31,6 +32,7 @@ #include #include #include "razor.h" +#include "import.h" /* Import a yum filelist as a razor package set. */ @@ -282,14 +284,76 @@ #define XML_BUFFER_SIZE 4096 +struct razor_stream { + z_stream strm; + void *in; + size_t in_length; +}; + +static int razor_stream_open(struct razor_stream *rs, const char *uri, + struct razor_error **error) +{ + rs->strm.zalloc = Z_NULL; + rs->strm.zfree = Z_NULL; + rs->strm.opaque = Z_NULL; + rs->strm.avail_in = 0; + rs->strm.next_in = Z_NULL; + + if (inflateInit2(&rs->strm, 15 + 16) != Z_OK) { + razor_set_error(error, RAZOR_GENERAL_ERROR, + RAZOR_GENERAL_ERROR_FAILED, uri, + "Failed to initialize inflator"); + return -1; + } + + rs->in = razor_uri_get_contents(uri, &rs->in_length, 0, error); + if (!rs->in) { + (void)inflateEnd(&rs->strm); + return -1; + } + + rs->strm.avail_in = rs->in_length; + rs->strm.next_in = rs->in; + + return 0; +} + +static ssize_t +razor_stream_read(struct razor_stream *rs, unsigned char *buf, size_t len) +{ + int r; + + rs->strm.avail_out = len; + rs->strm.next_out = buf; + + r = inflate(&rs->strm, Z_NO_FLUSH); + assert(r != Z_STREAM_ERROR); /* state not clobbered */ + switch (r) { + case Z_NEED_DICT: + case Z_DATA_ERROR: + case Z_MEM_ERROR: + return -1; + } + + return len - rs->strm.avail_out; +} + +static void razor_stream_close(struct razor_stream *rs) +{ + (void)inflateEnd(&rs->strm); + (void)razor_uri_free_contents(rs->in, rs->in_length); +} + struct razor_set * -razor_set_create_from_yum(void) +razor_set_create_from_yum(const char *yum_uri) { struct yum_context ctx={0}; + char *uri; void *buf; - int len; - gzFile primary, filelists; + ssize_t len; XML_ParsingStatus status; + struct razor_error *error = NULL; + struct razor_stream primary, filelists; ctx.importer = razor_importer_create(); ctx.state = YUM_STATE_BEGIN; @@ -310,16 +374,37 @@ XML_SetCharacterDataHandler(ctx.filelists_parser, yum_character_data); - primary = gzopen("primary.xml.gz", "rb"); - if (primary == NULL) { - perror("primary.xml.gz"); + uri = razor_path_relative_to_uri(yum_uri, "repodata/primary.xml.gz", + &error); + if (!uri) { + fprintf(stderr, "%s: %s\n", yum_uri, + razor_error_get_msg(error)); + razor_error_free(error); return NULL; } - filelists = gzopen("filelists.xml.gz", "rb"); - if (filelists == NULL) { - perror("filelists.xml.gz"); + if (razor_stream_open(&primary, uri, &error)) { + fprintf(stderr, "%s: %s\n", uri, razor_error_get_msg(error)); + free(uri); return NULL; } + free(uri); + + uri = razor_path_relative_to_uri(yum_uri, "repodata/filelists.xml.gz", + &error); + if (!uri) { + razor_stream_close(&primary); + fprintf(stderr, "%s: %s\n", yum_uri, + razor_error_get_msg(error)); + razor_error_free(error); + return NULL; + } + if (razor_stream_open(&filelists, uri, &error)) { + razor_stream_close(&primary); + fprintf(stderr, "%s: %s\n", uri, razor_error_get_msg(error)); + free(uri); + return NULL; + } + free(uri); ctx.current_parser = ctx.primary_parser; @@ -336,9 +421,11 @@ buf = XML_GetBuffer(ctx.current_parser, XML_BUFFER_SIZE); if (ctx.current_parser == ctx.primary_parser) - len = gzread(primary, buf, XML_BUFFER_SIZE); + len = razor_stream_read(&primary, buf, + XML_BUFFER_SIZE); else - len = gzread(filelists, buf, XML_BUFFER_SIZE); + len = razor_stream_read(&filelists, buf, + XML_BUFFER_SIZE); if (len < 0) { fprintf(stderr, "couldn't read input: %s\n", @@ -357,8 +444,8 @@ XML_ParserFree(ctx.primary_parser); XML_ParserFree(ctx.filelists_parser); - gzclose(primary); - gzclose(filelists); + razor_stream_close(&primary); + razor_stream_close(&filelists); printf ("\nsaving\n"); return razor_importer_finish(ctx.importer); diff -r 008c75a5e08d -r 48e45439fd9a src/import.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/import.h Mon Jul 04 13:04:19 2016 +0100 @@ -0,0 +1,9 @@ +#ifndef _RAZOR_IMPORT_H_ +#define _RAZOR_IMPORT_H_ + +#include "razor.h" + +struct razor_set *razor_set_create_from_yum(const char *yum_uri); +struct razor_set *razor_set_create_from_rpmdb(void); + +#endif /* _RAZOR_IMPORT_H_ */ diff -r 008c75a5e08d -r 48e45439fd9a src/main.c --- a/src/main.c Mon Jul 04 10:48:18 2016 +0100 +++ b/src/main.c Mon Jul 04 13:04:19 2016 +0100 @@ -46,6 +46,7 @@ #include #include #include "razor.h" +#include "import.h" static const char system_repo_filename[] = "system.rzdb"; static const char next_repo_filename[] = "system-next.rzdb"; @@ -614,140 +615,121 @@ RAZOR_PROPERTY_PROVIDES); } -#ifndef HAVE_CURL -static int -download_local(const char *url, const char *file) +#ifdef HAVE_CURL +struct get_contents_with_curl_baton { + void *buf; + size_t buflen; +}; + +static size_t +get_contents_with_curl_callback(void *contents, size_t size, size_t nmemb, + void *data) { - FILE *wfp, *rfp; - char buffer[256], *ptr, *local; - size_t nb, n; - struct razor_error *error = NULL; + size_t nb = size * nmemb; + struct get_contents_with_curl_baton *baton = data; - local = razor_path_from_uri(url, &error); - - if (local == NULL) { - fprintf(stderr, "%s: %s\n", file, razor_error_get_msg(error)); - razor_error_free(error); - return -1; - } else { - rfp = fopen(local, "rb"); - if (rfp == NULL) { - perror(local); - free(local); - return -1; - } - - wfp = fopen(file, "wb"); - if (wfp == NULL) { - perror(file); - fclose(rfp); - free(local); - return -1; - } - - while((nb = fread(buffer, 1, sizeof(buffer), rfp)) > 0) { - ptr = buffer; - while (nb > 0 && (n = fwrite(ptr, 1, nb, wfp)) > 0) { - ptr += n; - nb -= n; - } - - if (nb != 0) { - perror(file); - fclose(wfp); - fclose(rfp); - unlink(file); - free(local); - return -1; - } - } - - if (ferror(rfp)) { - perror(local); - fclose(wfp); - fclose(rfp); - unlink(file); - free(local); - return -1; - } - - fclose(wfp); - fclose(rfp); - free(local); + baton->buf = realloc(baton->buf, baton->buflen + nb); + if (!baton->buf) { + fprintf(stderr, "Not enough memory to read file\n"); return 0; } + + memcpy((char *)baton->buf + baton->buflen, contents, nb); + baton->buflen += nb; + + return nb; } -#endif /* !HAVE_CURL */ +static void *get_contents_with_curl(const char *uri, size_t *length, + int private, struct razor_error **error) +{ + CURL *curl; + char errbuf[256]; + CURLcode res; + struct get_contents_with_curl_baton baton = {0,}; + + curl = curl_easy_init(); + if (!curl) { + razor_set_error(error, RAZOR_GENERAL_ERROR, + RAZOR_GENERAL_ERROR_FAILED, uri, + "Failed to initialize libcurl"); + return NULL; + } + + curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, errbuf); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, + get_contents_with_curl_callback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&baton); + curl_easy_setopt(curl, CURLOPT_URL, uri); + curl_easy_setopt(curl, CURLOPT_USERAGENT, "razor/" VERSION); + res = curl_easy_perform(curl); + curl_easy_cleanup(curl); + + if (res != CURLE_OK) { + razor_set_error(error, RAZOR_GENERAL_ERROR, + RAZOR_GENERAL_ERROR_FAILED, uri, errbuf); + free(baton.buf); + return NULL; + } + + *length = baton.buflen; + return baton.buf; +} + +int free_contents_with_curl(void *addr, size_t length) +{ + free(addr); +} +#endif + +void init_uri_handler(void) +{ #ifdef HAVE_CURL -static int -show_progress(void *clientp, - double dltotal, double dlnow, double ultotal, double ulnow) -{ - const char *file = clientp; - - if (!dlnow < dltotal) - fprintf(stderr, "\rdownloading %s, %dkB/%dkB", - file, (int) dlnow / 1024, (int) dltotal / 1024); - - return 0; + struct razor_uri_vtable uri_vtable={0,}; + uri_vtable.structure_size=sizeof(uri_vtable); + uri_vtable.get_contents=get_contents_with_curl; + uri_vtable.free_contents=free_contents_with_curl; + razor_uri_set_vtable(NULL, &uri_vtable, NULL); +#endif } static int -download_with_curl(const char *url, const char *file) +download_if_missing(const char *uri, const char *file) { + int retval = 0; + struct stat buf; + void *contents; + size_t length; + struct razor_error *error = NULL; FILE *fp; - CURL *curl; - char error[256]; - CURLcode res; - long response; - - curl = curl_easy_init(); - if (curl == NULL) { - fprintf(stderr, - "%s: download manually (curl failed)\n", file); - return -1; - } - - curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, error); - curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0); - curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, show_progress); - curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, file); - - fp = fopen(file, "wb"); - if (fp == NULL) { - perror(file); - return -1; - } - curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp); - curl_easy_setopt(curl, CURLOPT_URL, url); - res = curl_easy_perform(curl); - fclose(fp); - putc('\n', stderr); - if (res != CURLE_OK) { - fprintf(stderr, "curl error: %s\n", error); - unlink(file); - return -1; - } - curl_easy_cleanup(curl); - - return 0; -} -#endif /* HAVE_CURL */ - -static int -download_if_missing(const char *url, const char *file) -{ - struct stat buf; if (stat(file, &buf) >= 0) return 0; -#ifndef HAVE_CURL - return download_local(url, file); -#else - return download_with_curl(url, file); -#endif + contents = razor_uri_get_contents(uri, &length, 0, &error); + + if (!contents) { + fprintf(stderr, "%s: %s\n", uri, razor_error_get_msg(error)); + razor_error_free(error); + return -1; + } + + fp = fopen(file, "wb"); + if (!fp) { + perror(file); + razor_uri_free_contents(contents, length); + return -1; + } + + if (fwrite(contents, 1, length, fp) != length) { + perror(file); + retval = -1; + } + + fclose(fp); + razor_uri_free_contents(contents, length); + + return retval; } #define YUM_URL "http://download.fedora.redhat.com" \ @@ -773,17 +755,9 @@ return 1; } - printf("downloading from '%s'.\n", yum_url); - snprintf(buffer, sizeof buffer, - "%s/repodata/primary.xml.gz", yum_url); - if (download_if_missing(buffer, "primary.xml.gz") < 0) - return -1; - snprintf(buffer, sizeof buffer, - "%s/repodata/filelists.xml.gz", yum_url); - if (download_if_missing(buffer, "filelists.xml.gz") < 0) - return -1; + printf("importing from '%s'.\n", yum_url); - set = razor_set_create_from_yum(); + set = razor_set_create_from_yum(yum_url); if (set == NULL) return 1; atomic = razor_atomic_open("Yum import repository"); @@ -1097,48 +1071,6 @@ return razor_concat(name, "-", v, ".", arch, ".rpm", NULL); } -static int -download_packages(struct razor_set *system, struct razor_set *next) -{ - struct razor_install_iterator *ii; - struct razor_package *package; - enum razor_install_action action; - const char *name, *version, *arch; - char *file, *url, *s; - int errors = 0, count; - - ii = razor_set_create_install_iterator(system, next); - while (razor_install_iterator_next(ii, &package, &action, &count)) { - if (action != RAZOR_INSTALL_ACTION_ADD) - continue; - - razor_package_get_details(next, package, - RAZOR_DETAIL_NAME, &name, - RAZOR_DETAIL_VERSION, &version, - RAZOR_DETAIL_ARCH, &arch, - RAZOR_DETAIL_LAST); - - s = rpm_filename(name, version, arch); - file = razor_concat("Packages/", s, NULL); - url = razor_path_relative_to_uri(yum_url, file, NULL); - free(file); - file = razor_concat("rpms/", s, NULL); - free(s); - if (download_if_missing(url, file) < 0) - errors++; - free(file); - free(url); - } - razor_install_iterator_destroy(ii); - - if (errors > 0) { - fprintf(stderr, "failed to download %d packages\n", errors); - return -1; - } - - return 0; -} - static struct razor_set * relocate_packages(struct razor_set *set, struct razor_atomic *atomic, struct razor_relocations *relocations) @@ -1179,12 +1111,7 @@ s = razor_concat("Packages/", filename, NULL); uri = razor_path_relative_to_uri(yum_url, s, NULL); free(s); - file = razor_concat("rpms/", filename, NULL); free(filename); - download_if_missing(uri, file); - free(uri); - uri = razor_path_to_uri(file); - free(file); rpm = razor_rpm_open(uri, &error); free(uri); if (rpm == NULL) { @@ -1255,14 +1182,15 @@ RAZOR_DETAIL_ARCH, &arch, RAZOR_DETAIL_LAST); + s = rpm_filename(name, version, arch); + file = razor_concat("Packages/", s, NULL); + free(s); + uri = razor_path_relative_to_uri(yum_url, file, NULL); + free(file); + if (stage & RAZOR_STAGE_SCRIPTS_PRE) - printf("install %s-%s\n", name, version); + printf("install %s\n", uri); - s = rpm_filename(name, version, arch); - file = razor_concat("rpms/", s, NULL); - free(s); - uri = razor_path_to_uri(file); - free(file); rpm = razor_rpm_open(uri, &error); free(uri); if (rpm == NULL) { @@ -1510,33 +1438,10 @@ } } - if (razor_atomic_create_dir(atomic, "file:rpms", - S_IRWXU | S_IRWXG | S_IRWXO) || - razor_atomic_commit(atomic)) { - fprintf(stderr, "%s\n", razor_atomic_get_error_msg(atomic)); - razor_transaction_destroy(trans); - razor_set_unref(upstream); - razor_root_close(root); - razor_atomic_destroy(atomic); - if (relocations) - razor_relocations_destroy(relocations); - return 1; - } - razor_atomic_destroy(atomic); next = razor_transaction_commit(trans); - if (download_packages(system, next) < 0) { - razor_set_unref(next); - razor_transaction_destroy(trans); - razor_set_unref(upstream); - razor_root_close(root); - if (relocations) - razor_relocations_destroy(relocations); - return 1; - } - retval = update_system(root, relocations, trans, next, do_update ? "Update" : "Install"); @@ -1975,6 +1880,8 @@ return 1; } + init_uri_handler(); + for (i = 0; i < ARRAY_SIZE(razor_commands); i++) if (strcmp(razor_commands[i].name, argv[main_optind]) == 0) return razor_commands[i].func(argc - main_optind, diff -r 008c75a5e08d -r 48e45439fd9a test/Makefile.am --- a/test/Makefile.am Mon Jul 04 10:48:18 2016 +0100 +++ b/test/Makefile.am Mon Jul 04 13:04:19 2016 +0100 @@ -4,38 +4,60 @@ if HAVE_LUA check_SCRIPTS += lua mult-install endif +if HAVE_ZIP + check_SCRIPTS += archive +endif +if HAVE_CURL +if !MSWIN_API + check_SCRIPTS += curl +endif +endif check_SCRIPTS += order non-ascii relative-root -relocate: relocate.sh primary.xml.gz +if !MSWIN_API + check_PROGRAMS = tftpd +endif + +if INSTALL_TEST_PROGRAMS + bin_PROGRAMS = $(check_PROGRAMS) +endif + +relocate: relocate.sh base/repodata/primary.xml.gz cp $(srcdir)/relocate.sh relocate -named-root: named-root.sh primary.xml.gz +named-root: named-root.sh base/repodata/primary.xml.gz cp $(srcdir)/named-root.sh named-root -remove: remove.sh primary.xml.gz +remove: remove.sh base/repodata/primary.xml.gz cp $(srcdir)/remove.sh remove update: update.sh base/repodata/primary.xml.gz updates/repodata/primary.xml.gz cp $(srcdir)/update.sh update -details: details.sh primary.xml.gz +details: details.sh base/repodata/primary.xml.gz cp $(srcdir)/details.sh details -order: order.sh primary.xml.gz +order: order.sh base/repodata/primary.xml.gz cp $(srcdir)/order.sh order -non-ascii: non-ascii.sh primary.xml.gz +non-ascii: non-ascii.sh base/repodata/primary.xml.gz cp $(srcdir)/non-ascii.sh non-ascii -relative-root: relative-root.sh primary.xml.gz +relative-root: relative-root.sh base/repodata/primary.xml.gz cp $(srcdir)/relative-root.sh relative-root -lua: lua.sh primary.xml.gz +lua: lua.sh base/repodata/primary.xml.gz cp $(srcdir)/lua.sh lua -mult-install: mult-install.sh primary.xml.gz +mult-install: mult-install.sh base/repodata/primary.xml.gz cp $(srcdir)/mult-install.sh mult-install +archive: archive.sh base.zip + cp $(srcdir)/archive.sh archive + +curl: curl.sh tftpd base/repodata/primary.xml.gz + cp $(srcdir)/curl.sh curl + base/repodata/primary.xml.gz: zsh.spec zsh2.spec zip.spec zap.spec \ filesystem.spec Makefile rm -rf rpmbuild base @@ -52,6 +74,9 @@ rm -rf rpmbuild createrepo --simple-md-filenames base +base.zip: base/repodata/primary.xml.gz + (cd base && zip -r ../$@ *) + updates/repodata/primary.xml.gz: zip.spec Makefile rm -rf rpmbuild updates mkdir -p rpmbuild/BUILD rpmbuild/RPMS @@ -62,11 +87,6 @@ rm -rf rpmbuild createrepo --simple-md-filenames updates -primary.xml.gz: base/repodata/primary.xml.gz - cp base/repodata/primary.xml.gz base/repodata/filelists.xml.gz . - rm -rf rpms - ln -s base/Packages rpms - TESTS = $(check_SCRIPTS) EXTRA_DIST = \ @@ -79,6 +99,8 @@ order.sh \ non-ascii.sh \ relative-root.sh \ + archive.sh \ + curl.sh \ mult-install.sh \ lua.sh \ remove.sh \ @@ -89,6 +111,7 @@ relocate.sh MOSTLYCLEANFILES = \ + base.zip \ primary.xml.gz \ filelists.xml.gz \ $(check_SCRIPTS) \ @@ -97,4 +120,4 @@ clean-local : rm -f *~ - rm -rf repodata rpms base updates + rm -rf base updates diff -r 008c75a5e08d -r 48e45439fd9a test/archive.sh --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test/archive.sh Mon Jul 04 13:04:19 2016 +0100 @@ -0,0 +1,29 @@ +#!/bin/sh +if [ $# -gt 0 ]; then + razor="$1" +else + razor=`pwd`/../src/razor +fi +check_file() +{ + (cd $scratchdir; $razor list-files) | grep -F -x "$1" > /dev/null + if [ $? -ne 0 ]; then + echo $1: Not in database >&2 + (cd $scratchdir; $razor list-files) >&2 + exit 1 + fi + if [ ! -e "$tmproot$1" ]; then + echo $1: Not in filesystem >&2 + ls -R "$tmproot" >&2 + exit 1 + fi +} +tmproot=`mktemp -dt` || exit 1 +export RAZOR_ROOT="file:$tmproot" +scratchdir=`mktemp -dt` || exit 1 +(cd $scratchdir; $razor init) || exit 1 +export YUM_URL="file:`pwd`/base.zip" +(cd $scratchdir; $razor import-yum) || exit 1 +(cd $scratchdir; $razor install zap) || exit 1 +check_file /usr/bin/zap +rm -rf "$scratchdir" "$tmproot" diff -r 008c75a5e08d -r 48e45439fd9a test/curl.sh --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test/curl.sh Mon Jul 04 13:04:19 2016 +0100 @@ -0,0 +1,50 @@ +#!/bin/bash +if [ $# -gt 0 ]; then + razor="$1" +else + razor=`pwd`/../src/razor +fi +check_file() +{ + $razor list-files | grep -F -x "$1" > /dev/null + if [ $? -ne 0 ]; then + echo $1: Not in database >&2 + $razor list-files >&2 + exit 1 + fi + if [ ! -e "$tmproot$1" ]; then + echo $1: Not in filesystem >&2 + ls -R "$tmproot" >&2 + exit 1 + fi +} +tmproot=`mktemp -dt` || exit 1 +export RAZOR_ROOT="file:$tmproot" +scratchdir=`mktemp -dt` || exit 1 +(cd base && ../tftpd $scratchdir/tftpd.pid $scratchdir/tftpd.port) || exit 1 +cd $scratchdir +port=`cat tftpd.port` +pid=`cat tftpd.pid` +export YUM_URL="tftp://localhost:$port/" +if $razor init; then + : +else + kill $pid + exit 1 +fi +if $razor import-yum; then + : +else + kill $pid + exit 1 +fi +if $razor install zap; then + : +else + kill $pid + exit 1 +fi +check_file /usr/bin/zap +rm -rf "$scratchdir" "$tmproot" +kill $pid +exit 0 diff -r 008c75a5e08d -r 48e45439fd9a test/tftpd.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test/tftpd.c Mon Jul 04 13:04:19 2016 +0100 @@ -0,0 +1,231 @@ +/* + * Copyright (C) 2016 J. Ali Harlow + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "config.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * A simple TFTP server suitable only for use on the loopback interface + * (it has no support for retransmitting lost packets). + */ + +char inbuf[SEGSIZE+4]; +char outbuf[SEGSIZE+4]; + +int send_error(int s, const struct sockaddr *client, socklen_t client_addr_len, + unsigned short code,const char *message) +{ + int r; + char *buf; + size_t len; + + len = 4 + strlen(message) + 1; + buf = malloc(len); + ((unsigned short *)buf)[0] = htons(ERROR); + ((unsigned short *)buf)[1] = htons(code); + strcpy(buf + 4, message); + r = sendto(s, buf, len, 0, client, client_addr_len); + free(buf); + + return r; +} + +int send_file(int s, const struct sockaddr *client, socklen_t client_addr_len, + const char *path, const char *mode) +{ + const char *dotdot, *p; + FILE *fp; + size_t nb; + int block = 0; + + if (strcasecmp(mode, "octet")) { + send_error(s, client, client_addr_len, EBADOP, "Bad mode"); + return -1; + } + + if (!*path || *path == '/') { + send_error(s, client, client_addr_len, EACCESS, + "Access denied"); + return -1; + } + + for (p = path; *p;) { + dotdot = strstr(p, ".."); + if (!dotdot) + break; + if ((dotdot == path || dotdot[-1] == '/') && + (dotdot[2] == '/' || dotdot[2] == '\0')) { + send_error(s, client, client_addr_len, EACCESS, + "Access denied"); + return -1; + } + p = dotdot + 2; + } + + fp = fopen(path, "rb"); + + if (!fp) { + if (errno == ENOENT) + send_error(s, client, client_addr_len, ENOTFOUND, + strerror(errno)); + else if (errno == EACCES) + send_error(s, client, client_addr_len, EACCESS, + strerror(errno)); + else + send_error(s, client, client_addr_len, EUNDEF, + strerror(errno)); + return -1; + } + + ((unsigned short *)outbuf)[0] = htons(DATA); + + while((nb = fread(outbuf + 4, 1, SEGSIZE, fp)) >= 0) + { + ((unsigned short *)outbuf)[1] = htons(++block); + if (sendto(s, outbuf, nb + 4, 0, client, client_addr_len) < 0) { + perror("sendto"); + fclose(fp); + return -1; + } + + /* Discard ACKs */ + (void)recvfrom(s, inbuf, sizeof(inbuf), 0, NULL, NULL); + + if (nb < SEGSIZE) + break; + } + + fclose(fp); + + return 0; +} + +void serve(int s) +{ + char *filename, *mode; + struct sockaddr_storage client; + socklen_t client_addr_len; + ssize_t nb; + + for(;;) { + client_addr_len = sizeof(client); + nb = recvfrom(s, inbuf, sizeof(inbuf), 0, + (struct sockaddr *)&client, &client_addr_len); + + if (nb < 0) { + perror("recvfrom"); + exit(1); + } + + if (nb >= 2) { + switch (ntohs(*(unsigned short *)inbuf)) { + case RRQ: + filename = inbuf + 2; + mode = memchr(filename, '\0', nb - 2); + if (!mode) { + send_error(s, + (struct sockaddr *)&client, + client_addr_len, EBADOP, + "Bad request"); + break; + } + mode++; + if (!memchr(mode, '\0', nb - (mode - inbuf))) { + send_error(s, + (struct sockaddr *)&client, + client_addr_len, EBADOP, + "Bad request"); + break; + } + send_file(s, (struct sockaddr *)&client, + client_addr_len, filename, mode); + break; + case WRQ: + send_error(s, (struct sockaddr *)&client, + client_addr_len, EACCESS, + "Access denied"); + break; + case DATA: + case ACK: + case ERROR: + break; + } + } + } +} + +main(int argc, char **argv) +{ + int s; + pid_t pid; + FILE *fp; + in_port_t port; + struct sockaddr_in sin = {0,}; + + s = socket(AF_INET, SOCK_DGRAM, 0); + if (s < 0) { + perror("socket"); + exit(1); + } + + sin.sin_family = AF_INET; + sin.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + + for (port = 0;; port++) { + sin.sin_port = htons(IPPORT_USERRESERVED + port); + if (!bind(s, (struct sockaddr *)&sin, sizeof(sin))) + break; + if (errno != EADDRINUSE || port >= 1023) { + perror("bind"); + exit(1); + } + } + + if (argc > 2) { + fp = fopen(argv[2], "w"); + if (fp) { + fprintf(fp, "%ld\n", (long)ntohs(sin.sin_port)); + fclose(fp); + } + + pid = fork(); + if (pid) { + fp = fopen(argv[1], "w"); + if (fp) { + fprintf(fp, "%ld\n", (long)pid); + fclose(fp); + } + exit(0); + } + } else + printf("%d\n", ntohs(sin.sin_port)); + + serve(s); + + exit(0); +}