Support downloading from local repository even without libcurl
authorJ. Ali Harlow <ali@juiblex.co.uk>
Thu Oct 09 17:27:41 2014 +0100 (2014-10-09)
changeset 455df914f383f5c
parent 454 56ff755c268c
child 456 bae5adee8c8c
Support downloading from local repository even without libcurl

Using the --url option of the razor executable, it is possible
to specify a yum repository on the local machine (eg., on installation
media) and import from there, eg.,:

C> razor --url file:///d:/ import-yum

This will be handled by libcurl if available but if not, an internal
copy routine will be used.

Note that if Microsoft's KTM implementation of atomic transactions is
used, then the current directory must support atomic transactions
(also improve error messages for this, and other, cases).
Makefile.am
configure.ac
librazor/Makefile.am
librazor/error.c
librazor/test-lua.c
librazor/types/Makefile.am
librazor/util.c
src/Makefile.am
src/import-yum.c
src/main.c
     1.1 --- a/Makefile.am	Sat Oct 04 18:12:58 2014 +0100
     1.2 +++ b/Makefile.am	Thu Oct 09 17:27:41 2014 +0100
     1.3 @@ -4,6 +4,8 @@
     1.4  
     1.5  ACLOCAL_AMFLAGS = -I gl/m4
     1.6  
     1.7 +DISTCHECK_CONFIGURE_FLAGS = --enable-tests
     1.8 +
     1.9  # Creating ChangeLog from git log (taken from cairo/Makefile.am):
    1.10  ChangeLog: $(srcdir)/ChangeLog
    1.11  
     2.1 --- a/configure.ac	Sat Oct 04 18:12:58 2014 +0100
     2.2 +++ b/configure.ac	Thu Oct 09 17:27:41 2014 +0100
     2.3 @@ -72,11 +72,18 @@
     2.4      AC_SUBST([RAZOR_HAVE_ATOMIC_ROLLBACK],['#undef RAZOR_HAVE_ATOMIC_ROLLBACK'])
     2.5  fi
     2.6  
     2.7 +AC_ARG_ENABLE([tests],
     2.8 +	      [AS_HELP_STRING([--enable-tests],
     2.9 +	      		      [install test programs])],
    2.10 +	      [],
    2.11 +	      [enable_tests=no])
    2.12 +AM_CONDITIONAL(INSTALL_TEST_PROGRAMS, test "$enable_tests" = "yes")
    2.13 +
    2.14  AC_MSG_CHECKING([for Microsoft Windows native API])
    2.15  case $host_os in
    2.16      *mingw*)	AC_DEFINE([MSWIN_API], 1,
    2.17  		  [Define to 1 to use Microsoft Windows native API.])
    2.18 -		EXTRA_LIBS='-lshell32 -lws2_32'
    2.19 +		EXTRA_LIBS='-lshlwapi -lshell32 -lws2_32'
    2.20  		mswin_api=yes;;
    2.21      *)		mswin_api=no;;
    2.22  esac
     3.1 --- a/librazor/Makefile.am	Sat Oct 04 18:12:58 2014 +0100
     3.2 +++ b/librazor/Makefile.am	Thu Oct 09 17:27:41 2014 +0100
     3.3 @@ -14,8 +14,13 @@
     3.4  	-DPACKAGE_LIB_DIR=\""$(libdir)"\"
     3.5  
     3.6  lib_LTLIBRARIES = librazor.la
     3.7 +check_PROGRAMS = test-pfu
     3.8  if HAVE_LUA
     3.9 -  check_PROGRAMS = test-lua
    3.10 +  check_PROGRAMS += test-lua
    3.11 +endif
    3.12 +
    3.13 +if INSTALL_TEST_PROGRAMS
    3.14 +  bin_PROGRAMS = $(check_PROGRAMS)
    3.15  endif
    3.16  
    3.17  librazorincludedir = $(includedir)/razor
    3.18 @@ -29,6 +34,7 @@
    3.19  	razor.c						\
    3.20  	root.c						\
    3.21  	util.c						\
    3.22 +	path.c						\
    3.23  	rpm.c						\
    3.24  	iterator.c					\
    3.25  	importer.c					\
    3.26 @@ -50,12 +56,18 @@
    3.27  librazor_la_LDFLAGS = -no-undefined -export-symbols-regex '^razor_' \
    3.28  	-version-info $(LT_CURRENT):$(LT_REVISION):$(LT_AGE)
    3.29  
    3.30 +test_pfu_SOURCES = test-pfu.c
    3.31 +test_pfu_LDADD = path.lo util.lo error.lo types/libtypes.la \
    3.32 +	../gl/libgnu.la $(INTLLIBS) $(EXTRA_LIBS)
    3.33 +
    3.34 +TESTS = test-pfu
    3.35 +
    3.36  if HAVE_LUA
    3.37    test_lua_SOURCES = test-lua.c
    3.38    test_lua_LDADD = lua.lo util.lo error.lo types/libtypes.la $(LUA_LIBS) \
    3.39  	../gl/libgnu.la $(INTLLIBS) $(EXTRA_LIBS)
    3.40  
    3.41 -  TESTS = test-lua
    3.42 +  TESTS += test-lua
    3.43  endif
    3.44  
    3.45  EXTRA_DIST = 			\
     4.1 --- a/librazor/error.c	Sat Oct 04 18:12:58 2014 +0100
     4.2 +++ b/librazor/error.c	Thu Oct 09 17:27:41 2014 +0100
     4.3 @@ -20,6 +20,9 @@
     4.4  
     4.5  #ifdef MSWIN_API
     4.6  #include <windows.h>
     4.7 +#ifndef ERROR_TRANSACTIONAL_OPEN_NOT_ALLOWED
     4.8 +#define ERROR_TRANSACTIONAL_OPEN_NOT_ALLOWED	6832L
     4.9 +#endif
    4.10  #endif
    4.11  #include <stdlib.h>
    4.12  #include <string.h>
    4.13 @@ -129,12 +132,28 @@
    4.14  	if (object)
    4.15  		error->object = razor_utf16_to_utf8(object, -1);
    4.16  
    4.17 -	FormatMessageW(FORMAT_MESSAGE_ALLOCATE_BUFFER|
    4.18 -		       FORMAT_MESSAGE_FROM_SYSTEM|FORMAT_MESSAGE_IGNORE_INSERTS,
    4.19 -		       NULL, err, MAKELANGID(LANG_NEUTRAL,SUBLANG_DEFAULT),
    4.20 -		       (LPWSTR)&buf, 0, NULL);
    4.21 -	error->str = razor_utf16_to_utf8(buf, -1);
    4.22 -	LocalFree(buf);
    4.23 +	switch(err) {
    4.24 +	case ERROR_TRANSACTIONAL_OPEN_NOT_ALLOWED:
    4.25 +		/*
    4.26 +		 * Attempting to include files in a transaction on a filesystem
    4.27 +		 * that doesn't support them (only NTFS?) produces this error
    4.28 +		 * for which the default text isn't very informative. Try and
    4.29 +		 * give more useful information.
    4.30 +		 */
    4.31 +		error->str = strdup("Not allowed (perhaps the filesystem "
    4.32 +				    "doesn't support transactions)");
    4.33 +		break;
    4.34 +	default:
    4.35 +		FormatMessageW(FORMAT_MESSAGE_ALLOCATE_BUFFER|
    4.36 +			       FORMAT_MESSAGE_FROM_SYSTEM|
    4.37 +			       FORMAT_MESSAGE_IGNORE_INSERTS,
    4.38 +			       NULL, err,
    4.39 +			       MAKELANGID(LANG_NEUTRAL,SUBLANG_DEFAULT),
    4.40 +			       (LPWSTR)&buf, 0, NULL);
    4.41 +		error->str = razor_utf16_to_utf8(buf, -1);
    4.42 +		LocalFree(buf);
    4.43 +		break;
    4.44 +	}
    4.45  
    4.46  	return error;
    4.47  }
     5.1 --- a/librazor/test-lua.c	Sat Oct 04 18:12:58 2014 +0100
     5.2 +++ b/librazor/test-lua.c	Thu Oct 09 17:27:41 2014 +0100
     5.3 @@ -91,8 +91,8 @@
     5.4  	}
     5.5  	fprintf(fp, "#!" LUA_BINARY "\n"
     5.6  	  "print('Abracadabra!')\n");
     5.7 -	fchmod(fileno(fp), S_IRUSR | S_IWUSR | S_IXUSR);
     5.8  	fclose(fp);
     5.9 +	chmod(s, S_IRUSR | S_IWUSR | S_IXUSR);
    5.10  	free(s);
    5.11  
    5.12  	script = razor_file_get_contents(test_file, &len, 0, &error);
     6.1 --- a/librazor/types/Makefile.am	Sat Oct 04 18:12:58 2014 +0100
     6.2 +++ b/librazor/types/Makefile.am	Thu Oct 09 17:27:41 2014 +0100
     6.3 @@ -4,6 +4,10 @@
     6.4  check_PROGRAMS = test-hashtable test-graph test-deque
     6.5  LDADD = libtypes.la
     6.6  
     6.7 +if INSTALL_TEST_PROGRAMS
     6.8 +  bin_PROGRAMS = $(check_PROGRAMS)
     6.9 +endif
    6.10 +
    6.11  libtypes_la_SOURCES =           \
    6.12  	array.c			\
    6.13  	deque.c			\
     7.1 --- a/librazor/util.c	Sat Oct 04 18:12:58 2014 +0100
     7.2 +++ b/librazor/util.c	Thu Oct 09 17:27:41 2014 +0100
     7.3 @@ -344,24 +344,6 @@
     7.4  	return concat;
     7.5  }
     7.6  
     7.7 -/**
     7.8 - * razor_path_add_root:
     7.9 - *
    7.10 - * Adds a root to a path. path must be an absolute pathname. In POSIX
    7.11 - * environments this is equivalent to the concationation of root and path.
    7.12 - * In Microsoft Windows an adjustment may need to be made for a drive letter
    7.13 - * in path (which will be dropped).
    7.14 - *
    7.15 - * Returns: The new pathname.
    7.16 - **/
    7.17 -RAZOR_EXPORT char *razor_path_add_root(const char *path, const char *root)
    7.18 -{
    7.19 -	if (root && *root)
    7.20 -		return razor_concat(root, SKIP_DRIVE_LETTER(path), NULL);
    7.21 -	else
    7.22 -		return strdup(path);
    7.23 -}
    7.24 -
    7.25  RAZOR_EXPORT const char *razor_system_arch(void)
    7.26  {
    7.27  #ifdef MSWIN_API
    7.28 @@ -394,12 +376,15 @@
    7.29  	int n;
    7.30  	char *utf8;
    7.31  
    7.32 +	if (len == 0)
    7.33 +		return strdup("");
    7.34 +
    7.35  	n = WideCharToMultiByte(CP_UTF8, 0, utf16, len, NULL, 0, NULL, NULL);
    7.36 -	if (len >= 0 && utf16[len])
    7.37 +	if (len > 0)
    7.38  		n++;
    7.39  	utf8 = malloc(n);
    7.40  	(void)WideCharToMultiByte(CP_UTF8, 0, utf16, len, utf8, n, NULL, NULL);
    7.41 -	if (len >= 0 && utf16[len])
    7.42 +	if (len > 0)
    7.43  		utf8[n - 1] = 0;
    7.44  
    7.45  	return utf8;
    7.46 @@ -410,12 +395,17 @@
    7.47  	int n;
    7.48  	wchar_t *utf16;
    7.49  
    7.50 +	if (len == 0) {
    7.51 +		utf16 = calloc(1, sizeof(wchar_t));
    7.52 +		return utf16;
    7.53 +	}
    7.54 +
    7.55  	n = MultiByteToWideChar(CP_UTF8, 0, utf8, len, NULL, 0);
    7.56 -	if (len >= 0 && utf8[len])
    7.57 +	if (len > 0)
    7.58  		n++;
    7.59  	utf16 = malloc(n * sizeof(wchar_t));
    7.60  	(void)MultiByteToWideChar(CP_UTF8, 0, utf8, len, utf16, n);
    7.61 -	if (len >= 0 && utf8[len])
    7.62 +	if (len > 0)
    7.63  		utf16[n - 1] = 0;
    7.64  
    7.65  	return utf16;
     8.1 --- a/src/Makefile.am	Sat Oct 04 18:12:58 2014 +0100
     8.2 +++ b/src/Makefile.am	Thu Oct 09 17:27:41 2014 +0100
     8.3 @@ -19,6 +19,11 @@
     8.4  noinst_PROGRAMS = rpm
     8.5  check_PROGRAMS = test-driver
     8.6  
     8.7 +if INSTALL_TEST_PROGRAMS
     8.8 +  bin_PROGRAMS = $(check_PROGRAMS)
     8.9 +  pkgdata_DATA = test.xml
    8.10 +endif
    8.11 +
    8.12  razor_SOURCES = main.c import-yum.c
    8.13  if HAVE_RPMLIB
    8.14  razor_SOURCES += import-rpmdb.c
     9.1 --- a/src/import-yum.c	Sat Oct 04 18:12:58 2014 +0100
     9.2 +++ b/src/import-yum.c	Thu Oct 09 17:27:41 2014 +0100
     9.3 @@ -311,11 +311,15 @@
     9.4  				    yum_character_data);
     9.5  
     9.6  	primary = gzopen("primary.xml.gz", "rb");
     9.7 -	if (primary == NULL)
     9.8 +	if (primary == NULL) {
     9.9 +		perror("primary.xml.gz");
    9.10  		return NULL;
    9.11 +	}
    9.12  	filelists = gzopen("filelists.xml.gz", "rb");
    9.13 -	if (filelists == NULL)
    9.14 +	if (filelists == NULL) {
    9.15 +		perror("filelists.xml.gz");
    9.16  		return NULL;
    9.17 +	}
    9.18  
    9.19  	ctx.current_parser = ctx.primary_parser;
    9.20  
    10.1 --- a/src/main.c	Sat Oct 04 18:12:58 2014 +0100
    10.2 +++ b/src/main.c	Thu Oct 09 17:27:41 2014 +0100
    10.3 @@ -32,9 +32,16 @@
    10.4  #include <fcntl.h>
    10.5  #include <dirent.h>
    10.6  #include <limits.h>
    10.7 +#ifdef MSWIN_API
    10.8 +#include <windows.h>
    10.9 +#include <shlwapi.h>
   10.10 +#endif
   10.11  #ifdef HAVE_CURL
   10.12  #include <curl/curl.h>
   10.13  #endif
   10.14 +#if !defined(HAVE_CURL) && !defined(MSWIN_API)
   10.15 +#include <ctype.h>
   10.16 +#endif
   10.17  #include <fnmatch.h>
   10.18  #include <errno.h>
   10.19  #include <getopt.h>
   10.20 @@ -607,6 +614,71 @@
   10.21  				      RAZOR_PROPERTY_PROVIDES);
   10.22  }
   10.23  
   10.24 +#ifndef HAVE_CURL
   10.25 +static int
   10.26 +download_local(const char *url, const char *file)
   10.27 +{
   10.28 +	FILE *wfp, *rfp;
   10.29 +	char buffer[256], *ptr, *local;
   10.30 +	size_t nb, n;
   10.31 +
   10.32 +	local = razor_path_from_url(url);
   10.33 +
   10.34 +	if (local == NULL) {
   10.35 +		fprintf(stderr,
   10.36 +			"%s: download manually (curl not available)\n",
   10.37 +			file);
   10.38 +		return -1;
   10.39 +	} else {
   10.40 +		rfp = fopen(local, "rb");
   10.41 +		if (rfp == NULL) {
   10.42 +			perror(local);
   10.43 +			free(local);
   10.44 +			return -1;
   10.45 +		}
   10.46 +
   10.47 +		wfp = fopen(file, "wb");
   10.48 +		if (wfp == NULL) {
   10.49 +			perror(file);
   10.50 +			fclose(rfp);
   10.51 +			free(local);
   10.52 +			return -1;
   10.53 +		}
   10.54 +
   10.55 +		while((nb = fread(buffer, 1, sizeof(buffer), rfp)) > 0) {
   10.56 +			ptr = buffer;
   10.57 +			while (nb > 0 && (n = fwrite(ptr, 1, nb, wfp)) > 0) {
   10.58 +				ptr += n;
   10.59 +				nb -= n;
   10.60 +			}
   10.61 +
   10.62 +			if (nb != 0) {
   10.63 +				perror(file);
   10.64 +				fclose(wfp);
   10.65 +				fclose(rfp);
   10.66 +				unlink(file);
   10.67 +				free(local);
   10.68 +				return -1;
   10.69 +			}
   10.70 +		}
   10.71 +
   10.72 +		if (ferror(rfp)) {
   10.73 +			perror(local);
   10.74 +			fclose(wfp);
   10.75 +			fclose(rfp);
   10.76 +			unlink(file);
   10.77 +			free(local);
   10.78 +			return -1;
   10.79 +		}
   10.80 +
   10.81 +		fclose(wfp);
   10.82 +		fclose(rfp);
   10.83 +		free(local);
   10.84 +		return 0;
   10.85 +	}
   10.86 +}
   10.87 +#endif /* !HAVE_CURL */
   10.88 +
   10.89  #ifdef HAVE_CURL
   10.90  static int
   10.91  show_progress(void *clientp,
   10.92 @@ -620,65 +692,74 @@
   10.93  
   10.94  	return 0;
   10.95  }
   10.96 -#endif	/* HAVE_CURL */
   10.97  
   10.98  static int
   10.99 -download_if_missing(const char *url, const char *file)
  10.100 +download_with_curl(const char *url, const char *file)
  10.101  {
  10.102 -#ifndef HAVE_CURL
  10.103 -	return 1;
  10.104 -#else
  10.105 +	FILE *fp;
  10.106  	CURL *curl;
  10.107 -	struct stat buf;
  10.108  	char error[256];
  10.109 -	FILE *fp;
  10.110  	CURLcode res;
  10.111  	long response;
  10.112  
  10.113  	curl = curl_easy_init();
  10.114 -	if (curl == NULL)
  10.115 -		return 1;
  10.116 +	if (curl == NULL) {
  10.117 +		fprintf(stderr,
  10.118 +			"%s: download manually (curl failed)\n", file);
  10.119 +		return -1;
  10.120 +	}
  10.121  
  10.122  	curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, error);
  10.123  	curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0);
  10.124  	curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, show_progress);
  10.125  	curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, file);
  10.126  
  10.127 -	if (stat(file, &buf) < 0) {
  10.128 -		fp = fopen(file, "wb");
  10.129 -		if (fp == NULL) {
  10.130 -			fprintf(stderr,
  10.131 -				"failed to open %s for writing\n", file);
  10.132 -			return -1;
  10.133 -		}
  10.134 -		curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
  10.135 -		curl_easy_setopt(curl, CURLOPT_URL, url);
  10.136 -		res = curl_easy_perform(curl);
  10.137 -		fclose(fp);
  10.138 -		if (res != CURLE_OK) {
  10.139 -			fprintf(stderr, "curl error: %s\n", error);
  10.140 -			unlink(file);
  10.141 -			return -1;
  10.142 -		}
  10.143 -		res = curl_easy_getinfo(curl,
  10.144 -					CURLINFO_RESPONSE_CODE, &response);
  10.145 -		if (res != CURLE_OK) {
  10.146 -			fprintf(stderr, "curl error: %s\n", error);
  10.147 -                        unlink(file);
  10.148 -                        return -1;
  10.149 -		}
  10.150 -		if (response != 200) {
  10.151 -			fprintf(stderr, " - failed %ld\n", response);
  10.152 -                        unlink(file);
  10.153 -                        return -1;
  10.154 -		}
  10.155 -		fprintf(stderr, "\n");
  10.156 +	fp = fopen(file, "wb");
  10.157 +	if (fp == NULL) {
  10.158 +		perror(file);
  10.159 +		return -1;
  10.160  	}
  10.161 -
  10.162 +	curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
  10.163 +	curl_easy_setopt(curl, CURLOPT_URL, url);
  10.164 +	res = curl_easy_perform(curl);
  10.165 +	fclose(fp);
  10.166 +	if (res != CURLE_OK) {
  10.167 +		fprintf(stderr, "curl error: %s\n", error);
  10.168 +		unlink(file);
  10.169 +		return -1;
  10.170 +	}
  10.171 +	res = curl_easy_getinfo(curl,
  10.172 +				CURLINFO_RESPONSE_CODE, &response);
  10.173 +	if (res != CURLE_OK) {
  10.174 +		fprintf(stderr, "curl error: %s\n", error);
  10.175 +		unlink(file);
  10.176 +		return -1;
  10.177 +	}
  10.178 +	if (response != 200) {
  10.179 +		fprintf(stderr, " - failed %ld\n", response);
  10.180 +		unlink(file);
  10.181 +		return -1;
  10.182 +	}
  10.183 +	fprintf(stderr, "\n");
  10.184  	curl_easy_cleanup(curl);
  10.185  
  10.186  	return 0;
  10.187 +}
  10.188  #endif	/* HAVE_CURL */
  10.189 +
  10.190 +static int
  10.191 +download_if_missing(const char *url, const char *file)
  10.192 +{
  10.193 +	struct stat buf;
  10.194 +
  10.195 +	if (stat(file, &buf) >= 0)
  10.196 +		return 0;
  10.197 +
  10.198 +#ifndef HAVE_CURL
  10.199 +	return download_local(url, file);
  10.200 +#else
  10.201 +	return download_with_curl(url, file);
  10.202 +#endif
  10.203  }
  10.204  
  10.205  #define YUM_URL "http://download.fedora.redhat.com" \
  10.206 @@ -704,7 +785,7 @@
  10.207  		return 1;
  10.208  	}
  10.209  
  10.210 -	printf("downloading from %s.\n", yum_url);
  10.211 +	printf("downloading from '%s'.\n", yum_url);
  10.212  	snprintf(buffer, sizeof buffer,
  10.213  		 "%s/repodata/primary.xml.gz", yum_url);
  10.214  	if (download_if_missing(buffer, "primary.xml.gz") < 0)
  10.215 @@ -1450,7 +1531,6 @@
  10.216  		razor_transaction_destroy(trans);
  10.217  		razor_set_unref(upstream);
  10.218  		razor_root_close(root);
  10.219 -		razor_atomic_destroy(atomic);
  10.220  		if (relocations)
  10.221  			razor_relocations_destroy(relocations);
  10.222                  return 1;