From 1e868858fd2de0d1125e6191be5e28df22fe6665 Mon Sep 17 00:00:00 2001 From: Mark H Weaver Date: Tue, 27 Mar 2018 20:05:58 -0400 Subject: tests: Use invoke and return #t from all builders. * tests/packages.scm ("package-source-derivation, snippet", "trivial") ("trivial with local file as input", "trivial with source") ("trivial with system-dependent input", "trivial with #:allowed-references") ("--search-paths with pattern", "--search-paths with single-item search path") ("replacement also grafted"): In the builders, raise an exception on errors and otherwise return #t. Use invoke. --- tests/packages.scm | 77 +++++++++++++++++++++++++++++++++++------------------- tests/profiles.scm | 6 +++-- 2 files changed, 54 insertions(+), 29 deletions(-) (limited to 'tests') diff --git a/tests/packages.scm b/tests/packages.scm index 9e19c3992..f1e7d3119 100644 --- a/tests/packages.scm +++ b/tests/packages.scm @@ -407,18 +407,23 @@ (%current-system))))) (arguments `(#:guile ,%bootstrap-guile + #:modules ((guix build utils)) #:builder - (let ((tar (assoc-ref %build-inputs "tar")) - (xz (assoc-ref %build-inputs "xz")) - (source (assoc-ref %build-inputs "source"))) - (and (zero? (system* tar "xvf" source - "--use-compress-program" xz)) - (string=? "guile" (readlink "bin/guile-rocks")) - (file-exists? "bin/scripts/compile.scm") - (let ((out (assoc-ref %outputs "out"))) - (call-with-output-file out - (lambda (p) - (display "OK" p)))))))))) + (begin + (use-modules (guix build utils)) + (let ((tar (assoc-ref %build-inputs "tar")) + (xz (assoc-ref %build-inputs "xz")) + (source (assoc-ref %build-inputs "source"))) + (invoke tar "xvf" source + "--use-compress-program" xz) + (unless (and (string=? "guile" (readlink "bin/guile-rocks")) + (file-exists? "bin/scripts/compile.scm")) + (error "the snippet apparently failed")) + (let ((out (assoc-ref %outputs "out"))) + (call-with-output-file out + (lambda (p) + (display "OK" p)))) + #t)))))) (drv (package-derivation %store package)) (out (derivation->output-path drv))) (and (build-derivations %store (list (pk 'snippet-drv drv))) @@ -486,7 +491,8 @@ (mkdir %output) (call-with-output-file (string-append %output "/test") (lambda (p) - (display '(hello guix) p)))))))) + (display '(hello guix) p))) + #t))))) (d (package-derivation %store p))) (and (build-derivations %store (list d)) (let ((p (pk 'drv d (derivation->output-path d)))) @@ -500,8 +506,10 @@ (source #f) (arguments `(#:guile ,%bootstrap-guile - #:builder (copy-file (assoc-ref %build-inputs "input") - %output))) + #:builder (begin + (copy-file (assoc-ref %build-inputs "input") + %output) + #t))) (inputs `(("input" ,i))))) (d (package-derivation %store p))) (and (build-derivations %store (list d)) @@ -516,8 +524,10 @@ (source i) (arguments `(#:guile ,%bootstrap-guile - #:builder (copy-file (assoc-ref %build-inputs "source") - %output))))) + #:builder (begin + (copy-file (assoc-ref %build-inputs "source") + %output) + #t))))) (d (package-derivation %store p))) (and (build-derivations %store (list d)) (let ((p (derivation->output-path d))) @@ -530,11 +540,14 @@ (source #f) (arguments `(#:guile ,%bootstrap-guile + #:modules ((guix build utils)) #:builder - (let ((out (assoc-ref %outputs "out")) - (bash (assoc-ref %build-inputs "bash"))) - (zero? (system* bash "-c" - (format #f "echo hello > ~a" out)))))) + (begin + (use-modules (guix build utils)) + (let ((out (assoc-ref %outputs "out")) + (bash (assoc-ref %build-inputs "bash"))) + (invoke bash "-c" + (format #f "echo hello > ~a" out)))))) (inputs `(("bash" ,(search-bootstrap-binary "bash" (%current-system))))))) (d (package-derivation %store p))) @@ -554,7 +567,8 @@ (mkdir %output) ;; The reference to itself isn't allowed so building it ;; should fail. - (symlink %output (string-append %output "/self"))))))) + (symlink %output (string-append %output "/self")) + #t))))) (d (package-derivation %store p))) (guard (c ((nix-protocol-error? c) #t)) (build-derivations %store (list d)) @@ -766,7 +780,9 @@ (inherit p1r) (name "p1") (replacement p1r) (arguments `(#:guile ,%bootstrap-guile - #:builder (mkdir (assoc-ref %outputs "out")))))) + #:builder (begin + (mkdir (assoc-ref %outputs "out")) + #t))))) (p2r (dummy-package "P2" (build-system trivial-build-system) (inputs `(("p1" ,p1))) @@ -786,7 +802,8 @@ (mkdir out) (chdir out) (symlink (assoc-ref %build-inputs "p1") - "p1")))))) + "p1") + #t))))) (p3 (dummy-package "p3" (build-system trivial-build-system) (inputs `(("p2" ,p2))) @@ -796,7 +813,8 @@ (mkdir out) (chdir out) (symlink (assoc-ref %build-inputs "p2") - "p2"))))))) + "p2") + #t)))))) (lset= equal? (package-grafts %store p3) (list (graft @@ -990,7 +1008,8 @@ (call-with-output-file (string-append out "/xml/bar/baz/catalog.xml") (lambda (port) - (display "xml? wat?!" port))))))) + (display "xml? wat?!" port))) + #t)))) (synopsis #f) (description #f) (home-page #f) (license #f))) (p2 (package @@ -1001,7 +1020,9 @@ (build-system trivial-build-system) (arguments `(#:guile ,%bootstrap-guile - #:builder (mkdir (assoc-ref %outputs "out")))) + #:builder (begin + (mkdir (assoc-ref %outputs "out")) + #t))) (native-search-paths (package-native-search-paths libxml2)) (synopsis #f) (description #f) (home-page #f) (license #f))) @@ -1043,7 +1064,9 @@ (build-system trivial-build-system) (arguments `(#:guile ,%bootstrap-guile - #:builder (mkdir (assoc-ref %outputs "out")))) + #:builder (begin + (mkdir (assoc-ref %outputs "out")) + #t))) (native-search-paths (package-native-search-paths git)))) (prof1 (run-with-store %store (profile-derivation diff --git a/tests/profiles.scm b/tests/profiles.scm index 92eb08cb9..eba79d4e3 100644 --- a/tests/profiles.scm +++ b/tests/profiles.scm @@ -453,7 +453,8 @@ (mkdir (string-append out "/etc")) (call-with-output-file (string-append out "/etc/foo") (lambda (port) - (display "foo!" port)))))))) + (display "foo!" port))) + #t))))) (entry -> (package->manifest-entry thing)) (drv (profile-derivation (manifest (list entry)) #:hooks '() @@ -482,7 +483,8 @@ (symlink "foo" (string-append out "/etc")) (call-with-output-file (string-append out "/etc/bar") (lambda (port) - (display "foo!" port)))))))) + (display "foo!" port))) + #t))))) (entry -> (package->manifest-entry thing)) (drv (profile-derivation (manifest (list entry)) #:hooks '() -- cgit v1.3 From 2d2f98efb36db3f003d950a004806234962b4f4d Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Fri, 4 May 2018 15:05:05 +0200 Subject: guix build: Nicely report unbound variables with hints. * guix/ui.scm (print-unbound-variable-error): Add "error:" to the message. (report-unbound-variable-error): New procedure, with code formerly in 'report-load-error'. (report-load-error): Use it. (call-with-unbound-variable-handling): New procedure. (with-unbound-variable-handling): New macro. * guix/scripts/build.scm (options->derivations): Wrap body in 'with-unbound-variable-handling'. * tests/guix-build.sh (GUIX_PACKAGE_PATH): Add test. --- guix/scripts/build.scm | 76 ++++++++++++++++++++++++++------------------------ guix/ui.scm | 51 +++++++++++++++++++++++++-------- tests/guix-build.sh | 21 +++++++++++++- 3 files changed, 100 insertions(+), 48 deletions(-) (limited to 'tests') diff --git a/guix/scripts/build.scm b/guix/scripts/build.scm index 401087e83..4dd4fbccd 100644 --- a/guix/scripts/build.scm +++ b/guix/scripts/build.scm @@ -661,43 +661,47 @@ build." (define system (assoc-ref opts 'system)) (define graft? (assoc-ref opts 'graft?)) - (parameterize ((%graft? graft?)) - (append-map (match-lambda - ((? package? p) - (let ((p (or (and graft? (package-replacement p)) p))) - (match src - (#f - (list (package->derivation store p system))) - (#t - (match (package-source p) - (#f - (format (current-error-port) - (G_ "~a: warning: \ + ;; We may get 'unbound-variable' errors while evaluating the 'inputs' fields + ;; of user packages. Since 'guix build' is the primary tool for people + ;; testing new packages, report such errors gracefully. + (with-unbound-variable-handling + (parameterize ((%graft? graft?)) + (append-map (match-lambda + ((? package? p) + (let ((p (or (and graft? (package-replacement p)) p))) + (match src + (#f + (list (package->derivation store p system))) + (#t + (match (package-source p) + (#f + (format (current-error-port) + (G_ "~a: warning: \ package '~a' has no source~%") - (location->string (package-location p)) - (package-name p)) - '()) - (s - (list (package-source-derivation store s))))) - (proc - (map (cut package-source-derivation store <>) - (proc p)))))) - ((? derivation? drv) - (list drv)) - ((? procedure? proc) - (list (run-with-store store - (mbegin %store-monad - (set-guile-for-build (default-guile)) - (proc)) - #:system system))) - ((? gexp? gexp) - (list (run-with-store store - (mbegin %store-monad - (set-guile-for-build (default-guile)) - (gexp->derivation "gexp" gexp - #:system system)))))) - (map (cut transform store <>) - (options->things-to-build opts))))) + (location->string (package-location p)) + (package-name p)) + '()) + (s + (list (package-source-derivation store s))))) + (proc + (map (cut package-source-derivation store <>) + (proc p)))))) + ((? derivation? drv) + (list drv)) + ((? procedure? proc) + (list (run-with-store store + (mbegin %store-monad + (set-guile-for-build (default-guile)) + (proc)) + #:system system))) + ((? gexp? gexp) + (list (run-with-store store + (mbegin %store-monad + (set-guile-for-build (default-guile)) + (gexp->derivation "gexp" gexp + #:system system)))))) + (map (cut transform store <>) + (options->things-to-build opts)))))) (define (show-build-log store file urls) "Show the build log for FILE, falling back to remote logs from URLS if diff --git a/guix/ui.scm b/guix/ui.scm index 223d2eb2a..8d351607d 100644 --- a/guix/ui.scm +++ b/guix/ui.scm @@ -76,6 +76,7 @@ show-manifest-transaction call-with-error-handling with-error-handling + with-unbound-variable-handling leave-on-EPIPE read/eval read/eval-package-expression @@ -158,7 +159,7 @@ messages." ((proc message (variable) _ ...) ;; We can always omit PROC because when it's useful (i.e., different from ;; "module-lookup"), it gets displayed before. - (format port (G_ "~a: unbound variable") variable)) + (format port (G_ "error: ~a: unbound variable") variable)) (_ (default-printer)))) @@ -309,6 +310,21 @@ PORT." (- (terminal-columns) 5)))) (texi->plain-text message)))) +(define* (report-unbound-variable-error args #:key frame) + "Return the given unbound-variable error, where ARGS is the list of 'throw' +arguments." + (match args + ((key . args) + (print-exception (current-error-port) frame key args))) + (match args + (('unbound-variable proc message (variable) _ ...) + (match (known-variable-definition variable) + (#f + (display-hint (G_ "Did you forget a @code{use-modules} form?"))) + ((? module? module) + (display-hint (format #f (G_ "Did you forget @code{(use-modules ~a)}?") + (module-name module)))))))) + (define* (report-load-error file args #:optional frame) "Report the failure to load FILE, a user-provided Scheme file. ARGS is the list of arguments received by the 'throw' handler." @@ -329,16 +345,8 @@ ARGS is the list of arguments received by the 'throw' handler." (let ((loc (source-properties->location properties))) (format (current-error-port) (G_ "~a: error: ~a~%") (location->string loc) message))) - (('unbound-variable proc message (variable) _ ...) - (match args - ((key . args) - (print-exception (current-error-port) frame key args))) - (match (known-variable-definition variable) - (#f - (display-hint (G_ "Did you forget a @code{use-modules} form?"))) - (module - (display-hint (format #f (G_ "Did you forget @code{(use-modules ~a)}?") - (module-name module)))))) + (('unbound-variable _ ...) + (report-unbound-variable-error args #:frame frame)) (('srfi-34 obj) (if (message-condition? obj) (if (error-location? obj) @@ -379,6 +387,27 @@ exiting. ARGS is the list of arguments received by the 'throw' handler." (warning (G_ "failed to load '~a':~%") file) (apply display-error #f (current-error-port) args)))) +(define (call-with-unbound-variable-handling thunk) + (define tag + (make-prompt-tag "user-code")) + + (catch 'unbound-variable + (lambda () + (call-with-prompt tag + thunk + (const #f))) + (const #t) + (rec (handle-error . args) + (let* ((stack (make-stack #t handle-error tag)) + (frame (and stack (last-frame-with-source stack)))) + (report-unbound-variable-error args #:frame frame) + (exit 1))))) + +(define-syntax-rule (with-unbound-variable-handling exp ...) + "Capture 'unbound-variable' exceptions in the dynamic extent of EXP... and +report them in a user-friendly way." + (call-with-unbound-variable-handling (lambda () exp ...))) + (define (install-locale) "Install the current locale settings." (catch 'system-error diff --git a/tests/guix-build.sh b/tests/guix-build.sh index b84723fa4..92e729932 100644 --- a/tests/guix-build.sh +++ b/tests/guix-build.sh @@ -1,5 +1,5 @@ # GNU Guix --- Functional package management for GNU -# Copyright © 2012, 2013, 2014, 2016, 2017 Ludovic Courtès +# Copyright © 2012, 2013, 2014, 2016, 2017, 2018 Ludovic Courtès # # This file is part of GNU Guix. # @@ -138,6 +138,25 @@ test `guix build -d --sources=transitive foo \ | grep -e 'foo\.tar\.gz' -e 'bar\.tar\.gz' -e 'bar\.dat' \ | wc -l` -eq 3 + +# Unbound variables. +cat > "$module_dir/foo.scm"< "$module_dir/err" || true +grep "unbound" "$module_dir/err" # actual error +grep "forget.*(gnu packages base)" "$module_dir/err" # hint +rm -f "$module_dir"/* + # Should all return valid log files. drv="`guix build -d -e '(@@ (gnu packages bootstrap) %bootstrap-guile)'`" out="`guix build -e '(@@ (gnu packages bootstrap) %bootstrap-guile)'`" -- cgit v1.3 From df6f86a0cb652172329597701683cfa837ddced2 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Mon, 7 May 2018 10:41:33 +0200 Subject: pack: Adjust test to cope with GC'd profiles. Previous "test -x opt/gnu/bin/guile" would fail if the store item "opt/gnu/bin" points to had been GC'd. * tests/guix-pack.sh: Replace "test -x" with "test -L" to deal with store items that have been reclaimed. --- tests/guix-pack.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'tests') diff --git a/tests/guix-pack.sh b/tests/guix-pack.sh index 1b63b957b..d34f72015 100644 --- a/tests/guix-pack.sh +++ b/tests/guix-pack.sh @@ -41,12 +41,14 @@ guix pack --bootstrap guile-bootstrap # Build a tarball with a symlink. the_pack="`guix pack --bootstrap -S /opt/gnu/bin=bin guile-bootstrap`" -# Try to extract it. +# Try to extract it. Note: we cannot test whether /opt/gnu/bin/guile itself +# exists because /opt/gnu/bin may be an absolute symlink to a store item that +# has been GC'd. test_directory="`mktemp -d`" trap 'rm -rf "$test_directory"' EXIT cd "$test_directory" tar -xf "$the_pack" -test -x opt/gnu/bin/guile +test -L opt/gnu/bin is_available () { # Use the "type" shell builtin to see if the program is on PATH. -- cgit v1.3 From aad16cc1965ab3488449c262455eb29b15c77e95 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Mon, 7 May 2018 10:44:18 +0200 Subject: pack: Honor package transformation options. Previously they would silently be ignored. * guix/scripts/pack.scm (guix-pack)[manifest-from-args]: Add 'store' parameter. Call 'options->transformation' and use it. Move 'with-store' and 'parameterize' around the 'let'. * tests/guix-pack.sh: Add test using '--with-source'. --- guix/scripts/pack.scm | 67 ++++++++++++++++++++++++++++----------------------- tests/guix-pack.sh | 7 ++++++ 2 files changed, 44 insertions(+), 30 deletions(-) (limited to 'tests') diff --git a/guix/scripts/pack.scm b/guix/scripts/pack.scm index b90bc41bc..1f493d8a0 100644 --- a/guix/scripts/pack.scm +++ b/guix/scripts/pack.scm @@ -43,6 +43,7 @@ #:autoload (gnu packages guile) (guile2.0-json guile-json) #:use-module (srfi srfi-1) #:use-module (srfi srfi-9) + #:use-module (srfi srfi-26) #:use-module (srfi srfi-37) #:use-module (ice-9 match) #:export (compressor? @@ -397,9 +398,14 @@ Create a bundle of PACKAGE.\n")) (read/eval-package-expression exp)) (x #f))) - (define (manifest-from-args opts) - (let ((packages (filter-map maybe-package-argument opts)) - (manifest-file (assoc-ref opts 'manifest))) + (define (manifest-from-args store opts) + (let* ((transform (options->transformation opts)) + (packages (map (match-lambda + (((? package? package) output) + (list (transform store package) + output))) + (filter-map maybe-package-argument opts))) + (manifest-file (assoc-ref opts 'manifest))) (cond ((and manifest-file (not (null? packages))) (leave (G_ "both a manifest and a package list were given~%"))) @@ -409,33 +415,34 @@ Create a bundle of PACKAGE.\n")) (else (packages->manifest packages))))) (with-error-handling - (let* ((dry-run? (assoc-ref opts 'dry-run?)) - (manifest (manifest-from-args opts)) - (pack-format (assoc-ref opts 'format)) - (name (string-append (symbol->string pack-format) - "-pack")) - (target (assoc-ref opts 'target)) - (bootstrap? (assoc-ref opts 'bootstrap?)) - (compressor (if bootstrap? - bootstrap-xz - (assoc-ref opts 'compressor))) - (tar (if bootstrap? - %bootstrap-coreutils&co - tar)) - (symlinks (assoc-ref opts 'symlinks)) - (build-image (match (assq-ref %formats pack-format) - ((? procedure? proc) proc) - (#f - (leave (G_ "~a: unknown pack format") - format)))) - (localstatedir? (assoc-ref opts 'localstatedir?))) - (with-store store - (parameterize ((%graft? (assoc-ref opts 'graft?)) - (%guile-for-build (package-derivation - store - (if (assoc-ref opts 'bootstrap?) - %bootstrap-guile - (canonical-package guile-2.2))))) + (with-store store + (parameterize ((%graft? (assoc-ref opts 'graft?)) + (%guile-for-build (package-derivation + store + (if (assoc-ref opts 'bootstrap?) + %bootstrap-guile + (canonical-package guile-2.2)) + #:graft? (assoc-ref opts 'graft?)))) + (let* ((dry-run? (assoc-ref opts 'dry-run?)) + (manifest (manifest-from-args store opts)) + (pack-format (assoc-ref opts 'format)) + (name (string-append (symbol->string pack-format) + "-pack")) + (target (assoc-ref opts 'target)) + (bootstrap? (assoc-ref opts 'bootstrap?)) + (compressor (if bootstrap? + bootstrap-xz + (assoc-ref opts 'compressor))) + (tar (if bootstrap? + %bootstrap-coreutils&co + tar)) + (symlinks (assoc-ref opts 'symlinks)) + (build-image (match (assq-ref %formats pack-format) + ((? procedure? proc) proc) + (#f + (leave (G_ "~a: unknown pack format") + format)))) + (localstatedir? (assoc-ref opts 'localstatedir?))) ;; Set the build options before we do anything else. (set-build-options-from-command-line store opts) diff --git a/tests/guix-pack.sh b/tests/guix-pack.sh index d34f72015..ec56ac96a 100644 --- a/tests/guix-pack.sh +++ b/tests/guix-pack.sh @@ -83,3 +83,10 @@ guix pack --dry-run --bootstrap -f docker -S /opt/gnu=/ guile-bootstrap # Build a tarball pack of cross-compiled software. Use coreutils because # guile-bootstrap is not intended to be cross-compiled. guix pack --dry-run --bootstrap --target=arm-unknown-linux-gnueabihf coreutils + +# Make sure package transformation options are honored. +mkdir -p "$test_directory" +drv1="`guix pack -n guile 2>&1 | grep pack.*\.drv`" +drv2="`guix pack -n --with-source=guile=$test_directory guile 2>&1 | grep pack.*\.drv`" +test -n "$drv1" +test "$drv1" != "$drv2" -- cgit v1.3 From d26727a143fab0dded1dbc848899270dbacd5dad Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Mon, 7 May 2018 14:04:19 +0200 Subject: pack: Fix handling of '-e'. Fixes a regression introduced in aad16cc1965ab3488449c262455eb29b15c77e95. Reported by Julien Lepiller. * guix/scripts/pack.scm (guix-pack)[manifest-from-args]: In 'match-lambda', add clause for single packages. * tests/guix-pack.sh: Add test for '-e'. --- guix/scripts/pack.scm | 5 +++-- tests/guix-pack.sh | 7 +++++-- 2 files changed, 8 insertions(+), 4 deletions(-) (limited to 'tests') diff --git a/guix/scripts/pack.scm b/guix/scripts/pack.scm index 1f493d8a0..0e09a0149 100644 --- a/guix/scripts/pack.scm +++ b/guix/scripts/pack.scm @@ -402,8 +402,9 @@ Create a bundle of PACKAGE.\n")) (let* ((transform (options->transformation opts)) (packages (map (match-lambda (((? package? package) output) - (list (transform store package) - output))) + (list (transform store package) output)) + ((? package? package) + (list (transform store package) "out"))) (filter-map maybe-package-argument opts))) (manifest-file (assoc-ref opts 'manifest))) (cond diff --git a/tests/guix-pack.sh b/tests/guix-pack.sh index ec56ac96a..5584c10e0 100644 --- a/tests/guix-pack.sh +++ b/tests/guix-pack.sh @@ -35,8 +35,11 @@ export GUIX_BUILD_OPTIONS # Build a tarball with no compression. guix pack --compression=none --bootstrap guile-bootstrap -# Build a tarball (with compression). -guix pack --bootstrap guile-bootstrap +# Build a tarball (with compression). Check that '-e' works as well. +out1="`guix pack --bootstrap guile-bootstrap`" +out2="`guix pack --bootstrap -e '(@ (gnu packages bootstrap) %bootstrap-guile)'`" +test -n "$out1" +test "$out1" = "$out2" # Build a tarball with a symlink. the_pack="`guix pack --bootstrap -S /opt/gnu/bin=bin guile-bootstrap`" -- cgit v1.3 From ede121de426f9c56820852888a0b370f0ccbce49 Mon Sep 17 00:00:00 2001 From: Chris Marusich Date: Sun, 8 Apr 2018 16:51:42 -0700 Subject: guix: Separate the package name and version with "@", not "-". * guix/packages.scm (package-full-name): By default, use "@" to separate the package name and package version. Add an optional delimiter argument so that there is still a way to explicitly use a different delimiter. * gnu/packages/commencement.scm (gcc-boot0) : Adjust accordingly. * tests/graph.scm: Adjust accordingly. * tests/profiles.scm: Adjust accordingly. * NEWS: Mention the change. Fixes: . Reported by Pierre Neidhardt . --- NEWS | 5 +++++ gnu/packages/commencement.scm | 2 +- guix/packages.go.134WZR | 0 guix/packages.scm | 13 +++++++++---- tests/graph.scm | 2 +- tests/profiles.scm | 11 +++++------ 6 files changed, 21 insertions(+), 12 deletions(-) create mode 100644 guix/packages.go.134WZR (limited to 'tests') diff --git a/NEWS b/NEWS index 2c898e65f..ca57f5d1f 100644 --- a/NEWS +++ b/NEWS @@ -10,6 +10,11 @@ Copyright © 2013, 2014, 2015, 2016, 2017 Ludovic Courtès Please send Guix bug reports to bug-guix@gnu.org. +* Changes in 0.15.0 (since 0.14.0) +** Programming interfaces + +*** package-full-name (guix packages) now uses "@" as its delimiter. + () * Changes in 0.14.0 (since 0.13.0) ** Package management diff --git a/gnu/packages/commencement.scm b/gnu/packages/commencement.scm index 1026ee892..2791409bf 100644 --- a/gnu/packages/commencement.scm +++ b/gnu/packages/commencement.scm @@ -282,7 +282,7 @@ ;; Drop trailing letters, as gmp-6.0.0a unpacks ;; into gmp-6.0.0. `(symlink ,(string-trim-right - (package-full-name lib) + (package-full-name lib "-") char-set:letter) ,(package-name lib))) (list gmp-6.0 mpfr mpc)))) diff --git a/guix/packages.go.134WZR b/guix/packages.go.134WZR new file mode 100644 index 000000000..e69de29bb diff --git a/guix/packages.scm b/guix/packages.scm index b5c0b6044..e0ab72086 100644 --- a/guix/packages.scm +++ b/guix/packages.scm @@ -388,10 +388,11 @@ object." (define-condition-type &package-cross-build-system-error &package-error package-cross-build-system-error?) - -(define (package-full-name package) - "Return the full name of PACKAGE--i.e., `NAME-VERSION'." - (string-append (package-name package) "-" (package-version package))) +(define* (package-full-name package #:optional (delimiter "@")) + "Return the full name of PACKAGE--i.e., `NAME@VERSION'. By specifying +DELIMITER (a string), you can customize what will appear between the name and +the version. By default, DELIMITER is \"@\"." + (string-append (package-name package) delimiter (package-version package))) (define (%standard-patch-inputs) (let* ((canonical (module-ref (resolve-interface '(gnu packages base)) @@ -935,6 +936,10 @@ and return it." (($ name version source build-system args inputs propagated-inputs native-inputs self-native-input? outputs) + ;; Even though we prefer to use "@" to separate the package + ;; name from the package version in various user-facing parts + ;; of Guix, checkStoreName (in nix/libstore/store-api.cc) + ;; prohibits the use of "@", so use "-" instead. (or (make-bag build-system (string-append name "-" version) #:system system #:target target diff --git a/tests/graph.scm b/tests/graph.scm index 5faa19298..b86ae4a32 100644 --- a/tests/graph.scm +++ b/tests/graph.scm @@ -134,7 +134,7 @@ edges." (map (lambda (destination) (list "p-0.drv" (string-append - (package-full-name destination) + (package-full-name destination "-") ".drv"))) implicit))))))) diff --git a/tests/profiles.scm b/tests/profiles.scm index 92eb08cb9..8d3cfe91d 100644 --- a/tests/profiles.scm +++ b/tests/profiles.scm @@ -242,8 +242,8 @@ #:hooks '() #:locales? #t #:target target))) - (define (find-input name) - (let ((name (string-append name ".drv"))) + (define (find-input package) + (let ((name (string-append (package-full-name package "-") ".drv"))) (any (lambda (input) (let ((input (derivation-input-path input))) (and (string-suffix? name input) input))) @@ -252,12 +252,11 @@ ;; The inputs for grep and sed should be cross-build derivations, but that ;; for the glibc-utf8-locales should be a native build. (return (and (string=? (derivation-system drv) (%current-system)) - (string=? (find-input (package-full-name packages:grep)) + (string=? (find-input packages:grep) (derivation-file-name grep)) - (string=? (find-input (package-full-name packages:sed)) + (string=? (find-input packages:sed) (derivation-file-name sed)) - (string=? (find-input - (package-full-name packages:glibc-utf8-locales)) + (string=? (find-input packages:glibc-utf8-locales) (derivation-file-name locales)))))) (test-assert "package->manifest-entry defaults to \"out\"" -- cgit v1.3 From dac1c97d131d297134fa878ac240d9ec0127044b Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Sat, 28 Apr 2018 17:17:33 +0200 Subject: union: Add 'relative-file-name'. * guix/build/union.scm (%not-slash): New variable. (relative-file-name): New procedure. * tests/union.scm (test-relative-file-name): New macro and tests. --- guix/build/union.scm | 41 ++++++++++++++++++++++++++++++++++++++++- tests/union.scm | 18 ++++++++++++++++++ tests/utils.scm | 2 +- 3 files changed, 59 insertions(+), 2 deletions(-) (limited to 'tests') diff --git a/guix/build/union.scm b/guix/build/union.scm index 1179f1234..82d6199d9 100644 --- a/guix/build/union.scm +++ b/guix/build/union.scm @@ -27,7 +27,9 @@ #:use-module (rnrs io ports) #:export (union-build - warn-about-collision)) + warn-about-collision + + relative-file-name)) ;;; Commentary: ;;; @@ -174,4 +176,41 @@ returns #f, skip the faulty file altogether." (union-of-directories output (delete-duplicates inputs))) + +;;; +;;; Relative symlinks. +;;; + +(define %not-slash + (char-set-complement (char-set #\/))) + +(define (relative-file-name reference file) + "Given REFERENCE and FILE, both of which are absolute file names, return the +file name of FILE relative to REFERENCE. + + (relative-file-name \"/gnu/store/foo\" \"/gnu/store/bin/bar\") + => \"../bin/bar\" + +Note that this is from a purely lexical standpoint; conversely, \"..\" is +*not* resolved lexically on POSIX in the presence of symlinks." + (if (and (string-prefix? "/" file) (string-prefix? "/" reference)) + (let loop ((reference (string-tokenize reference %not-slash)) + (file (string-tokenize file %not-slash))) + (define (finish) + (string-join (append (make-list (length reference) "..") file) + "/")) + + (match reference + (() + (finish)) + ((head . tail) + (match file + (() + (finish)) + ((head* . tail*) + (if (string=? head head*) + (loop tail tail*) + (finish))))))) + file)) + ;;; union.scm ends here diff --git a/tests/union.scm b/tests/union.scm index aa95cae00..5a6a4033f 100644 --- a/tests/union.scm +++ b/tests/union.scm @@ -184,4 +184,22 @@ (file-is-directory? "bin") (eq? 'symlink (stat:type (lstat "bin/guile")))))))) +(letrec-syntax ((test-relative-file-name + (syntax-rules (=>) + ((_ (reference file => expected) rest ...) + (begin + (test-equal (string-append "relative-file-name " + reference " " file) + expected + (relative-file-name reference file)) + (test-relative-file-name rest ...))) + ((_) + #t)))) + (test-relative-file-name + ("/a/b" "/a/c/d" => "../c/d") + ("/a/b" "/a/b" => "") + ("/a/b" "/a" => "..") + ("/a/b" "/a/b/c/d" => "c/d") + ("/a/b/c" "/a/d/e/f" => "../../d/e/f"))) + (test-end) diff --git a/tests/utils.scm b/tests/utils.scm index 035886dd1..197182acf 100644 --- a/tests/utils.scm +++ b/tests/utils.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017 Ludovic Courtès +;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018 Ludovic Courtès ;;; Copyright © 2014 Eric Bavier ;;; Copyright © 2016 Mathieu Lirzin ;;; -- cgit v1.3 From e00ade3fb81f89cd7c030f998ccd3e07ef2628f0 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Sat, 28 Apr 2018 22:20:36 +0200 Subject: profiles: Optionally use relative file names for symlink targets. * guix/build/union.scm (symlink-relative): New procedure. * guix/build/profiles.scm: Re-export it. (build-profile): Add #:symlink and pass it to 'union-build'. * guix/profiles.scm (profile-derivation): Add #:relative-symlinks?. Pass #:symlink to 'build-profile'. * tests/profiles.scm ("profile-derivation relative symlinks, one entry") ("profile-derivation relative symlinks, two entries"): New tests. --- guix/build/profiles.scm | 14 +++++++++----- guix/build/union.scm | 9 ++++++++- guix/profiles.scm | 7 +++++++ tests/profiles.scm | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 70 insertions(+), 6 deletions(-) (limited to 'tests') diff --git a/guix/build/profiles.scm b/guix/build/profiles.scm index b4160fba1..819688a91 100644 --- a/guix/build/profiles.scm +++ b/guix/build/profiles.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2015, 2017 Ludovic Courtès +;;; Copyright © 2015, 2017, 2018 Ludovic Courtès ;;; ;;; This file is part of GNU Guix. ;;; @@ -24,6 +24,7 @@ #:use-module (ice-9 ftw) #:use-module (ice-9 match) #:use-module (ice-9 pretty-print) + #:re-export (symlink-relative) ;for convenience #:export (ensure-writable-directory build-profile)) @@ -129,12 +130,15 @@ instead make DIRECTORY a \"real\" directory containing symlinks." (apply throw args)))))) (define* (build-profile output inputs - #:key manifest search-paths) - "Build a user profile from INPUTS in directory OUTPUT. Write MANIFEST, an -sexp, to OUTPUT/manifest. Create OUTPUT/etc/profile with Bash definitions for --all the variables listed in SEARCH-PATHS." + #:key manifest search-paths + (symlink symlink)) + "Build a user profile from INPUTS in directory OUTPUT, using SYMLINK to +create symlinks. Write MANIFEST, an sexp, to OUTPUT/manifest. Create +OUTPUT/etc/profile with Bash definitions for -all the variables listed in +SEARCH-PATHS." ;; Make the symlinks. (union-build output inputs + #:symlink symlink #:log-port (%make-void-port "w")) ;; Store meta-data. diff --git a/guix/build/union.scm b/guix/build/union.scm index 82d6199d9..24b366af4 100644 --- a/guix/build/union.scm +++ b/guix/build/union.scm @@ -29,7 +29,8 @@ warn-about-collision - relative-file-name)) + relative-file-name + symlink-relative)) ;;; Commentary: ;;; @@ -213,4 +214,10 @@ Note that this is from a purely lexical standpoint; conversely, \"..\" is (finish))))))) file)) +(define (symlink-relative old new) + "Assuming both OLD and NEW are absolute file names, make NEW a symlink to +OLD, but using a relative file name." + (symlink (relative-file-name (dirname new) old) + new)) + ;;; union.scm ends here diff --git a/guix/profiles.scm b/guix/profiles.scm index 95dc9746b..c17961c98 100644 --- a/guix/profiles.scm +++ b/guix/profiles.scm @@ -1202,6 +1202,7 @@ the entries in MANIFEST." (hooks %default-profile-hooks) (locales? #t) (allow-collisions? #f) + (relative-symlinks? #f) system target) "Return a derivation that builds a profile (aka. 'user environment') with the given MANIFEST. The profile includes additional derivations returned by @@ -1213,6 +1214,9 @@ with a different version number.) When LOCALES? is true, the build is performed under a UTF-8 locale; this adds a dependency on the 'glibc-utf8-locales' package. +When RELATIVE-SYMLINKS? is true, use relative file names for symlink targets. +This is one of the things to do for the result to be relocatable. + When TARGET is true, it must be a GNU triplet, and the packages in MANIFEST are cross-built for TARGET." (mlet* %store-monad ((system (if system @@ -1275,6 +1279,9 @@ are cross-built for TARGET." (manifest-entries manifest)))))) (build-profile #$output '#$inputs + #:symlink #$(if relative-symlinks? + #~symlink-relative + #~symlink) #:manifest '#$(manifest->gexp manifest) #:search-paths search-paths)))) diff --git a/tests/profiles.scm b/tests/profiles.scm index 8d3cfe91d..c668c2b83 100644 --- a/tests/profiles.scm +++ b/tests/profiles.scm @@ -223,6 +223,52 @@ (string=? (dirname (readlink bindir)) (derivation->output-path guile)))))) +(test-assertm "profile-derivation relative symlinks, one entry" + (mlet* %store-monad + ((entry -> (package->manifest-entry %bootstrap-guile)) + (guile (package->derivation %bootstrap-guile)) + (drv (profile-derivation (manifest (list entry)) + #:relative-symlinks? #t + #:hooks '() + #:locales? #f)) + (profile -> (derivation->output-path drv)) + (bindir -> (string-append profile "/bin")) + (_ (built-derivations (list drv)))) + (return (and (file-exists? (string-append bindir "/guile")) + (string=? (readlink bindir) + (string-append "../" + (basename + (derivation->output-path guile)) + "/bin")))))) + +(unless (network-reachable?) (test-skip 1)) +(test-assertm "profile-derivation relative symlinks, two entries" + (mlet* %store-monad + ((gnu-make-boot0 -> (@@ (gnu packages commencement) gnu-make-boot0)) + (manifest -> (packages->manifest + (list %bootstrap-guile gnu-make-boot0))) + (guile (package->derivation %bootstrap-guile)) + (make (package->derivation gnu-make-boot0)) + (drv (profile-derivation manifest + #:relative-symlinks? #t + #:hooks '() + #:locales? #f)) + (profile -> (derivation->output-path drv)) + (bindir -> (string-append profile "/bin")) + (_ (built-derivations (list drv)))) + (return (and (file-exists? (string-append bindir "/guile")) + (file-exists? (string-append bindir "/make")) + (string=? (readlink (string-append bindir "/guile")) + (string-append "../../" + (basename + (derivation->output-path guile)) + "/bin/guile")) + (string=? (readlink (string-append bindir "/make")) + (string-append "../../" + (basename + (derivation->output-path make)) + "/bin/make")))))) + (test-assertm "profile-derivation, inputs" (mlet* %store-monad ((entry -> (package->manifest-entry packages:glibc "debug")) -- cgit v1.3 From 47a60325ca650e8fc1a291c8655b4297f4de8deb Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Wed, 2 May 2018 17:08:37 +0200 Subject: pack: Add '--relocatable'. * gnu/packages/aux-files/run-in-namespace.c: New file. * Makefile.am (AUX_FILES): Add it. * guix/scripts/pack.scm (): New record type. (c-compiler, bootstrap-c-compiler, c-compiler-compiler): New procedures. (self-contained-tarball): Use 'relative-file-name' for the SOURCE -> TARGET symlink. (docker-image): Add 'defmod' to please Geiser. (wrapped-package, map-manifest-entries): New procedures. (%options, show-help): Add --relocatable. (guix-pack): Honor it. --- Makefile.am | 3 +- doc/guix.texi | 42 +++++ gnu/packages/aux-files/run-in-namespace.c | 264 ++++++++++++++++++++++++++++++ guix/scripts/pack.scm | 182 +++++++++++++++++++- tests/guix-pack.sh | 10 +- 5 files changed, 488 insertions(+), 13 deletions(-) create mode 100644 gnu/packages/aux-files/run-in-namespace.c (limited to 'tests') diff --git a/Makefile.am b/Makefile.am index 2110371f7..38bd54cf4 100644 --- a/Makefile.am +++ b/Makefile.am @@ -274,7 +274,8 @@ AUX_FILES = \ gnu/packages/aux-files/linux-libre/4.4-i686.conf \ gnu/packages/aux-files/linux-libre/4.4-x86_64.conf \ gnu/packages/aux-files/linux-libre/4.1-i686.conf \ - gnu/packages/aux-files/linux-libre/4.1-x86_64.conf + gnu/packages/aux-files/linux-libre/4.1-x86_64.conf \ + gnu/packages/aux-files/run-in-namespace.c # Templates, examples. EXAMPLES = \ diff --git a/doc/guix.texi b/doc/guix.texi index 8b9f8721b..6aeb31377 100644 --- a/doc/guix.texi +++ b/doc/guix.texi @@ -2834,6 +2834,15 @@ guix pack -S /opt/gnu/bin=bin guile emacs geiser @noindent That way, users can happily type @file{/opt/gnu/bin/guile} and enjoy. +@cindex relocatable binaries, with @command{guix pack} +What if the recipient of your pack does not have root privileges on +their machine, and thus cannot unpack it in the root file system? In +that case, you will want to use the @code{--relocatable} option (see +below). This option produces @dfn{relocatable binaries}, meaning they +they can be placed anywhere in the file system hierarchy: in the example +above, users can unpack your tarball in their home directory and +directly run @file{./opt/gnu/bin/guile}. + Alternatively, you can produce a pack in the Docker image format using the following command: @@ -2867,6 +2876,39 @@ This produces a tarball that follows the Docker Image Specification}. @end table +@item --relocatable +@itemx -R +Produce @dfn{relocatable binaries}---i.e., binaries that can be placed +anywhere in the file system hierarchy and run from there. For example, +if you create a pack containing Bash with: + +@example +guix pack -R -S /mybin=bin bash +@end example + +@noindent +... you can copy that pack to a machine that lacks Guix, and from your +home directory as a normal user, run: + +@example +tar xf pack.tar.gz +./mybin/sh +@end example + +@noindent +In that shell, if you type @code{ls /gnu/store}, you'll notice that +@file{/gnu/store} shows up and contains all the dependencies of +@code{bash}, even though the machine actually lacks @file{/gnu/store} +altogether! That is probably the simplest way to deploy Guix-built +software on a non-Guix machine. + +There's a gotcha though: this technique relies on the @dfn{user +namespace} feature of the kernel Linux, which allows unprivileged users +to mount or change root. Old versions of Linux did not support it, and +some GNU/Linux distributions turn it off; on these systems, programs +from the pack @emph{will fail to run}, unless they are unpacked in the +root file system. + @item --expression=@var{expr} @itemx -e @var{expr} Consider the package @var{expr} evaluates to. diff --git a/gnu/packages/aux-files/run-in-namespace.c b/gnu/packages/aux-files/run-in-namespace.c new file mode 100644 index 000000000..d0ab05c5d --- /dev/null +++ b/gnu/packages/aux-files/run-in-namespace.c @@ -0,0 +1,264 @@ +/* GNU Guix --- Functional package management for GNU + Copyright (C) 2018 Ludovic Courtès + + This file is part of GNU Guix. + + GNU Guix is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, or (at + your option) any later version. + + GNU Guix is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNU Guix. If not, see . */ + +/* Make the given @WRAPPED_PROGRAM@ relocatable by executing it in a separate + mount namespace where the store is mounted in its right place. + + We would happily do that in Scheme using 'call-with-container'. However, + this very program needs to be relocatable, so it needs to be statically + linked, which complicates things (Guile's modules can hardly be "linked" + into a single executable.) */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Concatenate DIRECTORY, a slash, and FILE. Return the result, which the + caller must eventually free. */ +static char * +concat (const char *directory, const char *file) +{ + char *result = malloc (strlen (directory) + 2 + strlen (file)); + assert (result != NULL); + + strcpy (result, directory); + strcat (result, "/"); + strcat (result, file); + return result; +} + +static void +mkdir_p (const char *directory) +{ + if (strcmp (directory, "/") != 0) + { + char *parent = dirname (strdupa (directory)); + mkdir_p (parent); + int err = mkdir (directory, 0700); + if (err < 0 && errno != EEXIST) + assert_perror (errno); + } +} + +static void +rm_rf (const char *directory) +{ + DIR *stream = opendir (directory); + + for (struct dirent *entry = readdir (stream); + entry != NULL; + entry = readdir (stream)) + { + if (strcmp (entry->d_name, ".") == 0 + || strcmp (entry->d_name, "..") == 0) + continue; + + char *full = concat (directory, entry->d_name); + + int err = unlink (full); + if (err < 0) + { + if (errno == EISDIR) + /* Recurse (we expect a shallow directory structure so there's + little risk of stack overflow.) */ + rm_rf (full); + else + assert_perror (errno); + } + + free (full); + } + + closedir (stream); + + int err = rmdir (directory); + if (err < 0 && errno != ENOENT) + assert_perror (errno); +} + +/* Bind mount all the top-level entries in SOURCE to TARGET. */ +static void +bind_mount (const char *source, const char *target) +{ + DIR *stream = opendir (source); + + for (struct dirent *entry = readdir (stream); + entry != NULL; + entry = readdir (stream)) + { + /* XXX: Some file systems may not report a useful 'd_type'. Ignore them + for now. */ + assert (entry->d_type != DT_UNKNOWN); + + if (strcmp (entry->d_name, ".") == 0 + || strcmp (entry->d_name, "..") == 0) + continue; + + char *abs_source = concat (source, entry->d_name); + char *new_entry = concat (target, entry->d_name); + + if (entry->d_type == DT_LNK) + { + char target[PATH_MAX]; + + ssize_t result = readlink (abs_source, target, sizeof target - 1); + if (result > 0) + { + target[result] = '\0'; + int err = symlink (target, new_entry); + if (err < 0) + assert_perror (errno); + } + } + else + { + /* Create the mount point. */ + if (entry->d_type == DT_DIR) + { + int err = mkdir (new_entry, 0700); + if (err != 0) + assert_perror (errno); + } + else + close (open (new_entry, O_WRONLY | O_CREAT)); + + int err = mount (abs_source, new_entry, "none", + MS_BIND | MS_REC | MS_RDONLY, NULL); + + /* It used to be that only directories could be bind-mounted. Thus, + keep going if we fail to bind-mount a non-directory entry. + That's OK because regular files in the root file system are + usually uninteresting. */ + if (err != 0 && entry->d_type != DT_DIR) + assert_perror (errno); + + free (new_entry); + free (abs_source); + } + } + + closedir (stream); +} + + +int +main (int argc, char *argv[]) +{ + ssize_t size; + char self[PATH_MAX]; + size = readlink ("/proc/self/exe", self, sizeof self - 1); + assert (size > 0); + + /* SELF is something like "/home/ludo/.local/gnu/store/…-foo/bin/ls" and we + want to extract "/home/ludo/.local/gnu/store". */ + size_t index = strlen (self) + - strlen ("@WRAPPED_PROGRAM@") + + strlen ("@STORE_DIRECTORY@"); + char *store = strdup (self); + store[index] = '\0'; + + struct stat statbuf; + + /* If STORE is already at the "right" place, we can execute + @WRAPPED_PROGRAM@ right away. This is not just an optimization: it's + needed when running one of these wrappers from within an unshare'd + namespace, because 'unshare' fails with EPERM in that context. */ + if (strcmp (store, "@STORE_DIRECTORY@") != 0 + && lstat ("@WRAPPED_PROGRAM@", &statbuf) != 0) + { + /* Spawn @WRAPPED_PROGRAM@ in a separate namespace where STORE is + bind-mounted in the right place. */ + int err; + char *new_root = mkdtemp (strdup ("/tmp/guix-exec-XXXXXX")); + char *new_store = concat (new_root, "@STORE_DIRECTORY@"); + char *cwd = get_current_dir_name (); + + pid_t child = fork (); + switch (child) + { + case 0: + /* Unshare namespaces in the child and set up bind-mounts from + there. That way, bind-mounts automatically disappear when the + child exits, which simplifies cleanup for the parent. */ + err = unshare (CLONE_NEWNS | CLONE_NEWUSER); + if (err < 0) + { + fprintf (stderr, "%s: error: 'unshare' failed: %m\n", argv[0]); + fprintf (stderr, "\ +This may be because \"user namespaces\" are not supported on this system.\n\ +Consequently, we cannot run '@WRAPPED_PROGRAM@',\n\ +unless you move it to the '@STORE_DIRECTORY@' directory.\n\ +\n\ +Please refer to the 'guix pack' documentation for more information.\n"); + return EXIT_FAILURE; + } + + /* Note: Due to + we cannot make NEW_ROOT a tmpfs (which would have saved the need + for 'rm_rf'.) */ + bind_mount ("/", new_root); + mkdir_p (new_store); + err = mount (store, new_store, "none", MS_BIND | MS_REC | MS_RDONLY, + NULL); + if (err < 0) + assert_perror (errno); + + chdir (new_root); + err = chroot (new_root); + if (err < 0) + assert_perror (errno); + + /* Change back to where we were before chroot'ing. */ + chdir (cwd); + break; + case -1: + assert_perror (errno); + break; + default: + { + int status; + waitpid (child, &status, 0); + chdir ("/"); /* avoid EBUSY */ + rm_rf (new_root); + free (new_root); + exit (status); + } + } + } + + /* The executable is available under @STORE_DIRECTORY@, so we can now + execute it. */ + int err = execv ("@WRAPPED_PROGRAM@", argv); + if (err < 0) + assert_perror (errno); + + return EXIT_FAILURE; +} diff --git a/guix/scripts/pack.scm b/guix/scripts/pack.scm index 0e09a0149..db5609219 100644 --- a/guix/scripts/pack.scm +++ b/guix/scripts/pack.scm @@ -32,6 +32,8 @@ #:use-module (guix packages) #:use-module (guix profiles) #:use-module (guix derivations) + #:use-module (guix search-paths) + #:use-module (guix build-system gnu) #:use-module (guix scripts build) #:use-module (gnu packages) #:use-module (gnu packages bootstrap) @@ -100,11 +102,14 @@ with a properly initialized store database. SYMLINKS must be a list of (SOURCE -> TARGET) tuples denoting symlinks to be added to the pack." (define build - (with-imported-modules '((guix build utils) - (guix build store-copy) - (gnu build install)) + (with-imported-modules (source-module-closure + '((guix build utils) + (guix build union) + (guix build store-copy) + (gnu build install))) #~(begin (use-modules (guix build utils) + ((guix build union) #:select (relative-file-name)) (gnu build install) (srfi srfi-1) (srfi srfi-26) @@ -119,7 +124,8 @@ added to the pack." ((source '-> target) (let ((target (string-append #$profile "/" target))) `((directory ,(dirname source)) - (,source -> ,target)))))) + (,source + -> ,(relative-file-name (dirname source) target))))))) (define directives ;; Fully-qualified symlinks. @@ -217,11 +223,13 @@ the image." (('gnu rest ...) #t) (rest #f))) + (define defmod 'define-module) ;trick Geiser + (define config ;; (guix config) module for consumption by (guix gcrypt). (scheme-file "gcrypt-config.scm" #~(begin - (define-module (guix config) + (#$defmod (guix config) #:export (%libgcrypt)) ;; XXX: Work around . @@ -265,6 +273,150 @@ the image." build #:references-graphs `(("profile" ,profile)))) + +;;; +;;; Compiling C programs. +;;; + +;; A C compiler. That lowers to a single program that can be passed typical C +;; compiler flags, and it makes sure the whole toolchain is available. +(define-record-type + (%c-compiler toolchain guile) + c-compiler? + (toolchain c-compiler-toolchain) + (guile c-compiler-guile)) + +(define* (c-compiler #:optional inputs + #:key (guile (default-guile))) + (%c-compiler inputs guile)) + +(define (bootstrap-c-compiler) + "Return the C compiler that uses the bootstrap toolchain. This is used only +by '--bootstrap', for testing purposes." + (define bootstrap-toolchain + (list (first (assoc-ref %bootstrap-inputs "gcc")) + (first (assoc-ref %bootstrap-inputs "binutils")) + (first (assoc-ref %bootstrap-inputs "libc")))) + + (c-compiler bootstrap-toolchain + #:guile %bootstrap-guile)) + +(define-gexp-compiler (c-compiler-compiler (compiler ) system target) + "Lower COMPILER to a single script that does the right thing." + (define toolchain + (or (c-compiler-toolchain compiler) + (list (first (assoc-ref (standard-packages) "gcc")) + (first (assoc-ref (standard-packages) "ld-wrapper")) + (first (assoc-ref (standard-packages) "binutils")) + (first (assoc-ref (standard-packages) "libc")) + (gexp-input (first (assoc-ref (standard-packages) "libc")) + "static")))) + + (define inputs + (match (append-map package-propagated-inputs + (filter package? toolchain)) + (((labels things . _) ...) + (append toolchain things)))) + + (define search-paths + (cons $PATH + (append-map package-native-search-paths + (filter package? inputs)))) + + (define run + (with-imported-modules (source-module-closure + '((guix build utils) + (guix search-paths))) + #~(begin + (use-modules (guix build utils) (guix search-paths) + (ice-9 match)) + + (define (output-file args) + (let loop ((args args)) + (match args + (() "a.out") + (("-o" file _ ...) file) + ((head rest ...) (loop rest))))) + + (set-search-paths (map sexp->search-path-specification + '#$(map search-path-specification->sexp + search-paths)) + '#$inputs) + + (let ((output (output-file (command-line)))) + (apply invoke "gcc" (cdr (command-line))) + (invoke "strip" output))))) + + (when target + ;; TODO: Yep, we'll have to do it someday! + (leave (G_ "cross-compilation not implemented here; +please email '~a'~%") + (@ (guix config) %guix-bug-report-address))) + + (gexp->script "c-compiler" run + #:guile (c-compiler-guile compiler))) + + +;;; +;;; Wrapped package. +;;; + +(define* (wrapped-package package + #:optional (compiler (c-compiler))) + (define runner + (local-file (search-auxiliary-file "run-in-namespace.c"))) + + (define build + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils) + (ice-9 match)) + + (define (strip-store-prefix file) + ;; Given a file name like "/gnu/store/…-foo-1.2/bin/foo", return + ;; "/bin/foo". + (let* ((len (string-length (%store-directory))) + (base (string-drop file (+ 1 len)))) + (match (string-index base #\/) + (#f base) + (index (string-drop base index))))) + + (define (build-wrapper program) + ;; Build a user-namespace wrapper for PROGRAM. + (format #t "building wrapper for '~a'...~%" program) + (copy-file #$runner "run.c") + + (substitute* "run.c" + (("@WRAPPED_PROGRAM@") program) + (("@STORE_DIRECTORY@") (%store-directory))) + + (let* ((base (strip-store-prefix program)) + (result (string-append #$output "/" base))) + (mkdir-p (dirname result)) + (invoke #$compiler "-std=gnu99" "-static" "-Os" "-g0" "-Wall" + "run.c" "-o" result) + (delete-file "run.c"))) + + (setvbuf (current-output-port) + (cond-expand (guile-2.2 'line) + (else _IOLBF))) + (for-each build-wrapper + (append (find-files #$(file-append package "/bin")) + (find-files #$(file-append package "/sbin")) + (find-files #$(file-append package "/libexec"))))))) + + (computed-file (string-append (package-full-name package "-") "R") + build)) + +(define (map-manifest-entries proc manifest) + "Apply PROC to all the entries of MANIFEST and return a new manifest." + (make-manifest + (map (lambda (entry) + (manifest-entry + (inherit entry) + (item (proc (manifest-entry-item entry))))) + (manifest-entries manifest)))) + ;;; ;;; Command-line options. @@ -302,6 +454,9 @@ the image." (option '(#\f "format") #t #f (lambda (opt name arg result) (alist-cons 'format (string->symbol arg) result))) + (option '(#\R "relocatable") #f #f + (lambda (opt name arg result) + (alist-cons 'relocatable? #t result))) (option '(#\e "expression") #t #f (lambda (opt name arg result) (alist-cons 'expression arg result))) @@ -354,6 +509,8 @@ Create a bundle of PACKAGE.\n")) (display (G_ " -f, --format=FORMAT build a pack in the given FORMAT")) (display (G_ " + -R, --relocatable produce relocatable executables")) + (display (G_ " -e, --expression=EXPR consider the package EXPR evaluates to")) (display (G_ " -s, --system=SYSTEM attempt to build for SYSTEM--e.g., \"i686-linux\"")) @@ -417,6 +574,9 @@ Create a bundle of PACKAGE.\n")) (with-error-handling (with-store store + ;; Set the build options before we do anything else. + (set-build-options-from-command-line store opts) + (parameterize ((%graft? (assoc-ref opts 'graft?)) (%guile-for-build (package-derivation store @@ -425,7 +585,13 @@ Create a bundle of PACKAGE.\n")) (canonical-package guile-2.2)) #:graft? (assoc-ref opts 'graft?)))) (let* ((dry-run? (assoc-ref opts 'dry-run?)) - (manifest (manifest-from-args store opts)) + (relocatable? (assoc-ref opts 'relocatable?)) + (manifest (let ((manifest (manifest-from-args store opts))) + ;; Note: We cannot honor '--bootstrap' here because + ;; 'glibc-bootstrap' lacks 'libc.a'. + (if relocatable? + (map-manifest-entries wrapped-package manifest) + manifest))) (pack-format (assoc-ref opts 'format)) (name (string-append (symbol->string pack-format) "-pack")) @@ -444,12 +610,10 @@ Create a bundle of PACKAGE.\n")) (leave (G_ "~a: unknown pack format") format)))) (localstatedir? (assoc-ref opts 'localstatedir?))) - ;; Set the build options before we do anything else. - (set-build-options-from-command-line store opts) - (run-with-store store (mlet* %store-monad ((profile (profile-derivation manifest + #:relative-symlinks? relocatable? #:hooks (if bootstrap? '() %default-profile-hooks) diff --git a/tests/guix-pack.sh b/tests/guix-pack.sh index 5584c10e0..130389a7a 100644 --- a/tests/guix-pack.sh +++ b/tests/guix-pack.sh @@ -20,9 +20,9 @@ # Test the `guix pack' command-line utility. # -# A network connection is required to build %bootstrap-coreutils&co, -# which is required to run these tests with the --bootstrap option. -if ! guile -c '(getaddrinfo "www.gnu.org" "80" AI_NUMERICSERV)' 2> /dev/null; then +# The bootstrap binaries are needed to run these tests, which usually requires +# a network connection. +if ! guix build -q guile-bootstrap; then exit 77 fi @@ -87,6 +87,10 @@ guix pack --dry-run --bootstrap -f docker -S /opt/gnu=/ guile-bootstrap # guile-bootstrap is not intended to be cross-compiled. guix pack --dry-run --bootstrap --target=arm-unknown-linux-gnueabihf coreutils +# Likewise, 'guix pack -R' requires a full-blown toolchain (because +# 'glibc-bootstrap' lacks 'libc.a'), hence '--dry-run'. +guix pack -R --dry-run --bootstrap -S /mybin=bin guile-bootstrap + # Make sure package transformation options are honored. mkdir -p "$test_directory" drv1="`guix pack -n guile 2>&1 | grep pack.*\.drv`" -- cgit v1.3 From 437f62f02a1b8c89b8ab39ecd53926489fac42db Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Sun, 13 May 2018 12:16:08 +0200 Subject: utils: Add 'version-prefix?'. * guix/utils.scm (version-prefix?): New procedure. * tests/utils.scm ("version-prefix?"): New test. --- guix/utils.scm | 24 +++++++++++++++++++++++- tests/utils.scm | 6 ++++++ 2 files changed, 29 insertions(+), 1 deletion(-) (limited to 'tests') diff --git a/guix/utils.scm b/guix/utils.scm index 92e45de61..e9efea586 100644 --- a/guix/utils.scm +++ b/guix/utils.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017 Ludovic Courtès +;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018 Ludovic Courtès ;;; Copyright © 2013, 2014, 2015 Mark H Weaver ;;; Copyright © 2014 Eric Bavier ;;; Copyright © 2014 Ian Denhardt @@ -84,6 +84,7 @@ version-major+minor version-major guile-version>? + version-prefix? string-replace-substring arguments-from-environment-variable file-extension @@ -521,6 +522,27 @@ minor version numbers from version-string." (micro-version)) str)) +(define version-prefix? + (let ((not-dot (char-set-complement (char-set #\.)))) + (lambda (v1 v2) + "Return true if V1 is a version prefix of V2: + + (version-prefix? \"4.1\" \"4.16.2\") => #f + (version-prefix? \"4.1\" \"4.1.2\") => #t +" + (define (list-prefix? lst1 lst2) + (match lst1 + (() #t) + ((head1 tail1 ...) + (match lst2 + (() #f) + ((head2 tail2 ...) + (and (equal? head1 head2) + (list-prefix? tail1 tail2))))))) + + (list-prefix? (string-tokenize v1 not-dot) + (string-tokenize v2 not-dot))))) + (define (file-extension file) "Return the extension of FILE or #f if there is none." (let ((dot (string-rindex file #\.))) diff --git a/tests/utils.scm b/tests/utils.scm index 197182acf..3015b21b2 100644 --- a/tests/utils.scm +++ b/tests/utils.scm @@ -72,6 +72,12 @@ (test-assert "guile-version>? 10.5" (not (guile-version>? "10.5"))) +(test-assert "version-prefix?" + (and (version-prefix? "4.1" "4.1.2") + (version-prefix? "4.1" "4.1") + (not (version-prefix? "4.1" "4.16.2")) + (not (version-prefix? "4.1" "4")))) + (test-equal "string-tokenize*" '(("foo") ("foo" "bar" "baz") -- cgit v1.3 From 7acdecec9992b81485814f153effdcb097c1e7c4 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Fri, 18 May 2018 15:23:57 +0200 Subject: tests: Adjust to new "unbound variable" messages. This is a followup to 2d2f98efb36db3f003d950a004806234962b4f4d. * tests/guix-system.sh: Adjust regexps to match "error:". --- tests/guix-system.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'tests') diff --git a/tests/guix-system.sh b/tests/guix-system.sh index ff9114ab7..2b94bc051 100644 --- a/tests/guix-system.sh +++ b/tests/guix-system.sh @@ -125,9 +125,9 @@ else then # FIXME: With Guile 2.2.0 the error is reported on line 4. # See . - grep "$tmpfile:[49]:[0-9]\+: GRUB-config.*[Uu]nbound variable" "$errorfile" + grep "$tmpfile:[49]:[0-9]\+:.*GRUB-config.*[Uu]nbound variable" "$errorfile" else - grep "$tmpfile:9:[0-9]\+: GRUB-config.*[Uu]nbound variable" "$errorfile" + grep "$tmpfile:9:[0-9]\+:.*GRUB-config.*[Uu]nbound variable" "$errorfile" fi fi -- cgit v1.3 From ce9e684b8395fdd0c8217a2e4a1d2776f8ff5ea4 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Fri, 18 May 2018 17:18:33 +0200 Subject: tests: Skip 'tests/guix-pack.sh' when networking is missing. The test could fail because "static-binaries.tar.xz" is missing, for instance. * tests/guix-pack.sh: Require a network connection to be on the safe side. This reverts part of 47a60325ca650e8fc1a291c8655b4297f4de8deb. --- tests/guix-pack.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'tests') diff --git a/tests/guix-pack.sh b/tests/guix-pack.sh index 130389a7a..917d52451 100644 --- a/tests/guix-pack.sh +++ b/tests/guix-pack.sh @@ -20,9 +20,9 @@ # Test the `guix pack' command-line utility. # -# The bootstrap binaries are needed to run these tests, which usually requires -# a network connection. -if ! guix build -q guile-bootstrap; then +# A network connection is required to build %bootstrap-coreutils&co, +# which is required to run these tests with the --bootstrap option. +if ! guile -c '(getaddrinfo "www.gnu.org" "80" AI_NUMERICSERV)' 2> /dev/null; then exit 77 fi -- cgit v1.3 From 263c9941a1e523b360ca9f42d1ed6b11e6e6e285 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Fri, 18 May 2018 22:20:33 +0200 Subject: uuid: 'uuid' returns #f when 'string->uuid' returns #f. * gnu/system/uuid.scm (uuid): When STR is not a literal, return #f when 'string->uuid' returns #f. * tests/uuid.scm ("uuid, dynamic value"): New test. --- gnu/system/uuid.scm | 11 +++++++---- tests/uuid.scm | 9 ++++++++- 2 files changed, 15 insertions(+), 5 deletions(-) (limited to 'tests') diff --git a/gnu/system/uuid.scm b/gnu/system/uuid.scm index 73695ddeb..f13960c3e 100644 --- a/gnu/system/uuid.scm +++ b/gnu/system/uuid.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2016, 2017 Ludovic Courtès +;;; Copyright © 2016, 2017, 2018 Ludovic Courtès ;;; Copyright © 2017 Danny Milosavljevic ;;; ;;; This file is part of GNU Guix. @@ -251,7 +251,8 @@ corresponding bytevector; otherwise return #f." (define-syntax uuid (lambda (s) - "Return the UUID object corresponding to the given UUID representation." + "Return the UUID object corresponding to the given UUID representation or +#f if the string could not be parsed." (syntax-case s (quote) ((_ str (quote type)) (and (string? (syntax->datum #'str)) @@ -266,9 +267,11 @@ corresponding bytevector; otherwise return #f." (string? (syntax->datum #'str)) #'(uuid str 'dce)) ((_ str) - #'(make-uuid 'dce (string->uuid str 'dce))) + #'(let ((bv (string->uuid str 'dce))) + (and bv (make-uuid 'dce bv)))) ((_ str type) - #'(make-uuid type (string->uuid str type)))))) + #'(let ((bv (string->uuid str type))) + (and bv (make-uuid type bv))))))) (define uuid->string ;; Convert the given bytevector or UUID object, to the corresponding UUID diff --git a/tests/uuid.scm b/tests/uuid.scm index 91a348249..260614f07 100644 --- a/tests/uuid.scm +++ b/tests/uuid.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2015, 2017 Ludovic Courtès +;;; Copyright © 2015, 2017, 2018 Ludovic Courtès ;;; ;;; This file is part of GNU Guix. ;;; @@ -57,6 +57,13 @@ "1234-ABCD" (uuid->string (uuid "1234-abcd" 'fat32))) +(test-assert "uuid, dynamic value" + (let* ((good "4dab5feb-d176-45de-b287-9b0a6e4c01cb") + (bad (string-drop good 3))) + (and (uuid? (uuid good)) + (string=? good (uuid->string (uuid good))) + (not (uuid bad))))) + (test-assert "uuid=?" (and (uuid=? (uuid-bytevector (uuid "1234-abcd" 'fat32)) (uuid "1234-abcd" 'fat32)) -- cgit v1.3 From 7874bbbb9f09cc14ea3e179fd0fa10da5f90cfc7 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Wed, 16 May 2018 10:05:24 +0200 Subject: records: Insert record type ABI checks in constructors. * guix/records.scm (print-record-abi-mismatch-error): New procedure. : Add 'set-exception-printer!' call. (current-abi-identifier, abi-check): New procedures. (make-syntactic-constructor): Add #:abi-cookie parameter. Insert calls to 'abi-check'. (define-record-type*)[compute-abi-cookie]: New procedure. Use it and emit a definition of the 'current-abi-identifier' for TYPE. * tests/records.scm ("ABI checks"): New test. --- guix/records.scm | 54 +++++++++++++++++++++++++++++++++++++++++++++++++++--- tests/records.scm | 30 +++++++++++++++++++++++++++++- 2 files changed, 80 insertions(+), 4 deletions(-) (limited to 'tests') diff --git a/guix/records.scm b/guix/records.scm index c02395f2a..c71cfcfe3 100644 --- a/guix/records.scm +++ b/guix/records.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017 Ludovic Courtès +;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018 Ludovic Courtès ;;; ;;; This file is part of GNU Guix. ;;; @@ -52,13 +52,45 @@ ((weird _ ...) ;weird! (syntax-violation name "invalid field specifier" #'weird))))) +(define (print-record-abi-mismatch-error port key args + default-printer) + (match args + ((rtd . _) + ;; The source file where this exception is thrown must be recompiled. + (format port "ERROR: ~a: record ABI mismatch; recompilation needed" + rtd)))) + +(set-exception-printer! 'record-abi-mismatch-error + print-record-abi-mismatch-error) + +(define (current-abi-identifier type) + "Return an identifier unhygienically derived from TYPE for use as its +\"current ABI\" variable." + (let ((type-name (syntax->datum type))) + (datum->syntax + type + (string->symbol + (string-append "% " (symbol->string type-name) + " abi-cookie"))))) + +(define (abi-check type cookie) + "Return syntax that checks that the current \"application binary +interface\" (ABI) for TYPE is equal to COOKIE." + (with-syntax ((current-abi (current-abi-identifier type))) + #`(unless (eq? current-abi #,cookie) + (throw 'record-abi-mismatch-error #,type)))) + (define-syntax make-syntactic-constructor (syntax-rules () "Make the syntactic constructor NAME for TYPE, that calls CTOR, and expects all of EXPECTED fields to be initialized. DEFAULTS is the list of FIELD/DEFAULT-VALUE tuples, THUNKED is the list of identifiers of thunked -fields, and DELAYED is the list of identifiers of delayed fields." +fields, and DELAYED is the list of identifiers of delayed fields. + +ABI-COOKIE is the cookie (an integer) against which to check the run-time ABI +of TYPE matches the expansion-time ABI." ((_ type name ctor (expected ...) + #:abi-cookie abi-cookie #:thunked thunked #:delayed delayed #:innate innate @@ -130,6 +162,7 @@ fields, and DELAYED is the list of identifiers of delayed fields." (syntax-case s (inherit expected ...) ((_ (inherit orig-record) (field value) (... ...)) #`(let* #,(field-bindings #'((field value) (... ...))) + #,(abi-check #'type abi-cookie) #,(record-inheritance #'orig-record #'((field value) (... ...))))) ((_ (field value) (... ...)) @@ -144,6 +177,7 @@ fields, and DELAYED is the list of identifiers of delayed fields." (cond ((lset= eq? fields '(expected ...)) #`(let* #,(field-bindings #'((field value) (... ...))) + #,(abi-check #'type abi-cookie) (ctor #,@(map field-value '(expected ...))))) ((pair? (lset-difference eq? fields '(expected ...))) @@ -270,6 +304,16 @@ inherited." ;; The real value of that field is a promise, so force it. (force (real-get x))))))) + (define (compute-abi-cookie field-specs) + ;; Compute an "ABI cookie" for the given FIELD-SPECS. We use + ;; 'string-hash' because that's a better hash function that 'hash' on a + ;; list of symbols. + (syntax-case field-specs () + (((field get properties ...) ...) + (string-hash (object->string + (syntax->datum #'((field properties ...) ...))) + most-positive-fixnum)))) + (syntax-case s () ((_ type syntactic-ctor ctor pred (field get properties ...) ...) @@ -278,7 +322,8 @@ inherited." (delayed (filter-map delayed-field? field-spec)) (innate (filter-map innate-field? field-spec)) (defaults (filter-map field-default-value - #'((field properties ...) ...)))) + #'((field properties ...) ...))) + (cookie (compute-abi-cookie field-spec))) (with-syntax (((field-spec* ...) (map field-spec->srfi-9 field-spec)) ((thunked-field-accessor ...) @@ -298,10 +343,13 @@ inherited." (ctor field ...) pred field-spec* ...) + (define #,(current-abi-identifier #'type) + #,cookie) thunked-field-accessor ... delayed-field-accessor ... (make-syntactic-constructor type syntactic-ctor ctor (field ...) + #:abi-cookie #,cookie #:thunked #,thunked #:delayed #,delayed #:innate #,innate diff --git a/tests/records.scm b/tests/records.scm index d6d27bb96..80e08a9a5 100644 --- a/tests/records.scm +++ b/tests/records.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2012, 2013, 2014, 2015, 2016 Ludovic Courtès +;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2018 Ludovic Courtès ;;; ;;; This file is part of GNU Guix. ;;; @@ -288,6 +288,34 @@ (and (string-match "extra.*initializer.*baz" message) (eq? proc 'foo))))) +(test-assert "ABI checks" + (let ((module (test-module))) + (eval '(begin + (define-record-type* foo make-foo + foo? + (bar foo-bar (default 42))) + + (define (make-me-a-record) (foo))) + module) + (unless (eval '(foo? (make-me-a-record)) module) + (error "what?" (eval '(make-me-a-record) module))) + + ;; Redefine with an additional field. + (eval '(define-record-type* foo make-foo + foo? + (baz foo-baz) + (bar foo-bar (default 42))) + module) + + ;; Now 'make-me-a-record' is out of sync because it does an + ;; 'allocate-struct' that corresponds to the previous definition of . + (catch 'record-abi-mismatch-error + (lambda () + (eval '(foo? (make-me-a-record)) module) + #f) + (lambda (key rtd . _) + (eq? rtd (eval ' module)))))) + (test-equal "recutils->alist" '((("Name" . "foo") ("Version" . "0.1") -- cgit v1.3 From 9ceeca0880248eee8c2796f5e6ebc172b95277f7 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Fri, 18 May 2018 15:04:36 +0200 Subject: system: Remove uses of the 'title' field of . * gnu/system/install.scm (installation-os): Remove uses of the 'title' field of 'file-system'; use 'file-system-label' as appropriate. * gnu/system/vm.scm (system-disk-image, system-qemu-image): Likewise. * gnu/tests.scm (%simple-os): Likewise. * gnu/tests/install.scm (%minimal-os, %minimal-extlinux-os) (%minimal-os-on-vda, %separate-home-os, %separate-store-os) (%raid-root-os, %encrypted-root-os, %btrfs-root-os): Likewise. * gnu/build/shepherd.scm (default-mounts)[tmpfs]: Likewise. * tests/guix-system.sh: Likewise. * tests/system.scm (%root-fs): Likewise. ("operating-system-boot-mapped-devices, implicit dependency"): Likewise. --- gnu/build/shepherd.scm | 3 +-- gnu/system/install.scm | 4 +--- gnu/system/vm.scm | 2 -- gnu/tests.scm | 3 +-- gnu/tests/install.scm | 26 ++++++++------------------ tests/guix-system.sh | 9 +++------ tests/system.scm | 6 ++---- 7 files changed, 16 insertions(+), 37 deletions(-) (limited to 'tests') diff --git a/gnu/build/shepherd.scm b/gnu/build/shepherd.scm index c955e3c83..f38325992 100644 --- a/gnu/build/shepherd.scm +++ b/gnu/build/shepherd.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2017 Ludovic Courtès +;;; Copyright © 2017, 2018 Ludovic Courtès ;;; ;;; This file is part of GNU Guix. ;;; @@ -55,7 +55,6 @@ (define (tmpfs directory) (file-system (device "none") - (title 'device) (mount-point directory) (type "tmpfs") (check? #f))) diff --git a/gnu/system/install.scm b/gnu/system/install.scm index 3efff915a..acc9f15e0 100644 --- a/gnu/system/install.scm +++ b/gnu/system/install.scm @@ -328,8 +328,7 @@ You have been warned. Thanks for being so brave.\x1b[0m ;; the appropriate one. (cons* (file-system (mount-point "/") - (device "GuixSD_image") - (title 'label) + (device (file-system-label "GuixSD_image")) (type "ext4")) ;; Make /tmp a tmpfs instead of keeping the overlayfs. This @@ -341,7 +340,6 @@ You have been warned. Thanks for being so brave.\x1b[0m (file-system (mount-point "/tmp") (device "none") - (title 'device) (type "tmpfs") (check? #f)) diff --git a/gnu/system/vm.scm b/gnu/system/vm.scm index 7f8014715..4cffc71d7 100644 --- a/gnu/system/vm.scm +++ b/gnu/system/vm.scm @@ -571,7 +571,6 @@ to USB sticks meant to be read-only." (file-systems (cons (file-system (mount-point "/") (device root-uuid) - (title 'uuid) (type file-system-type)) file-systems-to-keep))))) @@ -636,7 +635,6 @@ of the GNU system as described by OS." (file-systems (cons (file-system (mount-point "/") (device root-uuid) - (title 'uuid) (type file-system-type)) file-systems-to-keep))))) (mlet* %store-monad diff --git a/gnu/tests.scm b/gnu/tests.scm index 2aef370af..5d4a4f806 100644 --- a/gnu/tests.scm +++ b/gnu/tests.scm @@ -208,8 +208,7 @@ the system under test." (bootloader grub-bootloader) (target "/dev/sdX"))) (file-systems (cons (file-system - (device "my-root") - (title 'label) + (device (file-system-label "my-root")) (mount-point "/") (type "ext4")) %base-file-systems)) diff --git a/gnu/tests/install.scm b/gnu/tests/install.scm index e3bb1b46a..4764ffffd 100644 --- a/gnu/tests/install.scm +++ b/gnu/tests/install.scm @@ -66,8 +66,7 @@ (target "/dev/vdb"))) (kernel-arguments '("console=ttyS0")) (file-systems (cons (file-system - (device "my-root") - (title 'label) + (device (file-system-label "my-root")) (mount-point "/") (type "ext4")) %base-file-systems)) @@ -105,8 +104,7 @@ (target "/dev/vdb"))) (kernel-arguments '("console=ttyS0")) (file-systems (cons (file-system - (device "my-root") - (title 'label) + (device (file-system-label "my-root")) (mount-point "/") (type "ext4")) %base-file-systems)) @@ -351,8 +349,7 @@ per %test-installed-os, this test is expensive in terms of CPU and storage.") (target "/dev/vda"))) (kernel-arguments '("console=ttyS0")) (file-systems (cons (file-system - (device "my-root") - (title 'label) + (device (file-system-label "my-root")) (mount-point "/") (type "ext4")) %base-file-systems)) @@ -428,13 +425,11 @@ reboot\n") (target "/dev/vdb"))) (kernel-arguments '("console=ttyS0")) (file-systems (cons* (file-system - (device "my-root") - (title 'label) + (device (file-system-label "my-root")) (mount-point "/") (type "ext4")) (file-system (device "none") - (title 'device) (type "tmpfs") (mount-point "/home") (type "tmpfs")) @@ -488,13 +483,11 @@ partition. In particular, home directories must be correctly created (see (target "/dev/vdb"))) (kernel-arguments '("console=ttyS0")) (file-systems (cons* (file-system - (device "root-fs") - (title 'label) + (device (file-system-label "root-fs")) (mount-point "/") (type "ext4")) (file-system - (device "store-fs") - (title 'label) + (device (file-system-label "store-fs")) (mount-point "/gnu") (type "ext4")) %base-file-systems)) @@ -574,8 +567,7 @@ where /gnu lives on a separate partition.") (target "/dev/md0") (type raid-device-mapping)))) (file-systems (cons (file-system - (device "root-fs") - (title 'label) + (device (file-system-label "root-fs")) (mount-point "/") (type "ext4") (dependencies mapped-devices)) @@ -658,7 +650,6 @@ by 'mdadm'.") (type luks-device-mapping)))) (file-systems (cons (file-system (device "/dev/mapper/the-root-device") - (title 'device) (mount-point "/") (type "ext4")) %base-file-systems)) @@ -779,8 +770,7 @@ build (current-guix) and then store a couple of full system images.") (target "/dev/vdb"))) (kernel-arguments '("console=ttyS0")) (file-systems (cons (file-system - (device "my-root") - (title 'label) + (device (file-system-label "my-root")) (mount-point "/") (type "btrfs")) %base-file-systems)) diff --git a/tests/guix-system.sh b/tests/guix-system.sh index 2b94bc051..36ba5fbd5 100644 --- a/tests/guix-system.sh +++ b/tests/guix-system.sh @@ -111,8 +111,7 @@ cat > "$tmpfile" < +;;; Copyright © 2016, 2018 Ludovic Courtès ;;; Copyright © 2017 Tobias Geerinckx-Rice ;;; ;;; This file is part of GNU Guix. @@ -27,8 +27,7 @@ (define %root-fs (file-system - (device "my-root") - (title 'label) + (device (file-system-label "my-root")) (mount-point "/") (type "ext4"))) @@ -114,7 +113,6 @@ (inherit %os-with-mapped-device) (file-systems (cons (file-system (device "/dev/mapper/my-luks-device") - (title 'device) (mount-point "/") (type "ext4")) %base-file-systems))))) -- cgit v1.3 From a82eefe65a7319edd28034747b601c21da924f16 Mon Sep 17 00:00:00 2001 From: Ricardo Wurmus Date: Wed, 30 May 2018 09:55:28 +0200 Subject: tests: Fix arguments in pack test. This is a follow-up to commit 5ffac538aa604b71814ac74579626f0d3110b96e. * tests/pack.scm (self-contained-tarball): Adjust arguments to "self-contained-tarball". --- tests/pack.scm | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'tests') diff --git a/tests/pack.scm b/tests/pack.scm index 3bce71507..fcc53d12e 100644 --- a/tests/pack.scm +++ b/tests/pack.scm @@ -1,5 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2017 Ludovic Courtès +;;; Copyright © 2018 Ricardo Wurmus ;;; ;;; This file is part of GNU Guix. ;;; @@ -61,7 +62,7 @@ #:symlinks '(("/bin/Guile" -> "bin/guile")) #:compressor %gzip-compressor - #:tar %tar-bootstrap)) + #:archiver %tar-bootstrap)) (check (gexp->derivation "check-tarball" #~(let ((guile (string-append "." #$profile "/bin"))) -- cgit v1.3 From 44057a461b1fa8102938c4e0f54d7cbc9dd09b03 Mon Sep 17 00:00:00 2001 From: Ricardo Wurmus Date: Wed, 30 May 2018 09:55:28 +0200 Subject: tests: Fix arguments in pack test. This is a follow-up to commit 5ffac538aa604b71814ac74579626f0d3110b96e. * tests/pack.scm (self-contained-tarball): Adjust arguments to "self-contained-tarball". --- tests/pack.scm | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'tests') diff --git a/tests/pack.scm b/tests/pack.scm index 3bce71507..fcc53d12e 100644 --- a/tests/pack.scm +++ b/tests/pack.scm @@ -1,5 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2017 Ludovic Courtès +;;; Copyright © 2018 Ricardo Wurmus ;;; ;;; This file is part of GNU Guix. ;;; @@ -61,7 +62,7 @@ #:symlinks '(("/bin/Guile" -> "bin/guile")) #:compressor %gzip-compressor - #:tar %tar-bootstrap)) + #:archiver %tar-bootstrap)) (check (gexp->derivation "check-tarball" #~(let ((guile (string-append "." #$profile "/bin"))) -- cgit v1.3 From ccc951cab3172adfdaf6fd2dfa8f8cdb98358a69 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Fri, 1 Jun 2018 15:17:41 +0200 Subject: pack: Adjust test to expect relative symlinks. Reported by Chris Marusich . Fixes . * tests/pack.scm ("self-contained-tarball"): Rename 'guile' to 'bin'. Expect 'bin/Guile' to be a relative symlink. --- tests/pack.scm | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'tests') diff --git a/tests/pack.scm b/tests/pack.scm index fcc53d12e..d4596f863 100644 --- a/tests/pack.scm +++ b/tests/pack.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2017 Ludovic Courtès +;;; Copyright © 2017, 2018 Ludovic Courtès ;;; Copyright © 2018 Ricardo Wurmus ;;; ;;; This file is part of GNU Guix. @@ -65,17 +65,17 @@ #:archiver %tar-bootstrap)) (check (gexp->derivation "check-tarball" - #~(let ((guile (string-append "." #$profile "/bin"))) + #~(let ((bin (string-append "." #$profile "/bin"))) (setenv "PATH" (string-append #$%tar-bootstrap "/bin")) (system* "tar" "xvf" #$tarball) (mkdir #$output) (exit - (and (file-exists? (string-append guile "/guile")) + (and (file-exists? (string-append bin "/guile")) (string=? (string-append #$%bootstrap-guile "/bin") - (readlink guile)) - (string=? (string-append (string-drop guile 1) - "/guile") + (readlink bin)) + (string=? (string-append ".." #$profile + "/bin/guile") (readlink "bin/Guile")))))))) (built-derivations (list check)))) -- cgit v1.3 From 838e17d8050236a6d3ffde991fb0035412eb3046 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Mon, 28 May 2018 18:14:37 +0200 Subject: gexp: Add 'with-extensions'. * guix/gexp.scm ()[extensions]: New field. (gexp-attribute): New procedure. (gexp-modules): Write in terms of 'gexp-attribute'. (gexp-extensions): New procedure. (gexp->derivation): Add #:effective-version. [extension-flags]: New procedure. Honor extensions of EXP. (current-imported-extensions): New syntax parameter. (with-extensions): New macro. (gexp): Honor CURRENT-IMPORTED-EXTENSIONS. (compiled-modules): Add #:extensions and honor it. (load-path-expression): Likewise. (gexp->script, gexp->file): Honor extensions. * tests/gexp.scm (%extension-package): New variable. ("gexp-extensions & ungexp") ("gexp-extensions & ungexp-splicing") ("gexp-extensions and literal Scheme object") ("gexp->derivation & with-extensions") ("program-file & with-extensions"): New tests. * doc/guix.texi (G-Expressions): Document 'with-extensions'. --- .dir-locals.el | 1 + doc/guix.texi | 33 ++++++++++++ guix/gexp.scm | 168 ++++++++++++++++++++++++++++++++++++++++++--------------- tests/gexp.scm | 86 +++++++++++++++++++++++++++++ 4 files changed, 246 insertions(+), 42 deletions(-) (limited to 'tests') diff --git a/.dir-locals.el b/.dir-locals.el index dac6cb145..2db751ca2 100644 --- a/.dir-locals.el +++ b/.dir-locals.el @@ -73,6 +73,7 @@ (eval . (put 'run-with-state 'scheme-indent-function 1)) (eval . (put 'wrap-program 'scheme-indent-function 1)) (eval . (put 'with-imported-modules 'scheme-indent-function 1)) + (eval . (put 'with-extensions 'scheme-indent-function 1)) (eval . (put 'call-with-container 'scheme-indent-function 1)) (eval . (put 'container-excursion 'scheme-indent-function 1)) diff --git a/doc/guix.texi b/doc/guix.texi index 3b5078741..77bdaa50e 100644 --- a/doc/guix.texi +++ b/doc/guix.texi @@ -5064,6 +5064,23 @@ headers, which comes in handy in this case: @dots{}))) @end example +@cindex extensions, for gexps +@findex with-extensions +In the same vein, sometimes you want to import not just pure-Scheme +modules, but also ``extensions'' such as Guile bindings to C libraries +or other ``full-blown'' packages. Say you need the @code{guile-json} +package available on the build side, here's how you would do it: + +@example +(use-modules (gnu packages guile)) ;for 'guile-json' + +(with-extensions (list guile-json) + (gexp->derivation "something-with-json" + #~(begin + (use-modules (json)) + @dots{}))) +@end example + The syntactic form to construct gexps is summarized below. @deffn {Scheme Syntax} #~@var{exp} @@ -5147,6 +5164,18 @@ directly defined in @var{body}@dots{}, but not on those defined, say, in procedures called from @var{body}@dots{}. @end deffn +@deffn {Scheme Syntax} with-extensions @var{extensions} @var{body}@dots{} +Mark the gexps defined in @var{body}@dots{} as requiring +@var{extensions} in their build and execution environment. +@var{extensions} is typically a list of package objects such as those +defined in the @code{(gnu packages guile)} module. + +Concretely, the packages listed in @var{extensions} are added to the +load path while compiling imported modules in @var{body}@dots{}; they +are also added to the load path of the gexp returned by +@var{body}@dots{}. +@end deffn + @deffn {Scheme Procedure} gexp? @var{obj} Return @code{#t} if @var{obj} is a G-expression. @end deffn @@ -5161,6 +5190,7 @@ information about monads.) [#:hash #f] [#:hash-algo #f] @ [#:recursive? #f] [#:env-vars '()] [#:modules '()] @ [#:module-path @var{%load-path}] @ + [#:effective-version "2.2"] @ [#:references-graphs #f] [#:allowed-references #f] @ [#:disallowed-references #f] @ [#:leaked-env-vars #f] @ @@ -5181,6 +5211,9 @@ make @var{modules} available in the evaluation context of @var{exp}; the load path during the execution of @var{exp}---e.g., @code{((guix build utils) (guix build gnu-build-system))}. +@var{effective-version} determines the string to use when adding extensions of +@var{exp} (see @code{with-extensions}) to the search path---e.g., @code{"2.2"}. + @var{graft?} determines whether packages referred to by @var{exp} should be grafted when applicable. diff --git a/guix/gexp.scm b/guix/gexp.scm index fdfd73424..338c339da 100644 --- a/guix/gexp.scm +++ b/guix/gexp.scm @@ -33,6 +33,7 @@ #:export (gexp gexp? with-imported-modules + with-extensions gexp-input gexp-input? @@ -118,10 +119,11 @@ ;; "G expressions". (define-record-type - (make-gexp references modules proc) + (make-gexp references modules extensions proc) gexp? (references gexp-references) ;list of (modules gexp-self-modules) ;list of module names + (extensions gexp-self-extensions) ;list of lowerable things (proc gexp-proc)) ;procedure (define (write-gexp gexp port) @@ -492,19 +494,20 @@ whether this should be considered a \"native\" input or not." (set-record-type-printer! write-gexp-output) -(define (gexp-modules gexp) - "Return the list of Guile module names GEXP relies on. If (gexp? GEXP) is -false, meaning that GEXP is a plain Scheme object, return the empty list." +(define (gexp-attribute gexp self-attribute) + "Recurse on GEXP and the expressions it refers to, summing the items +returned by SELF-ATTRIBUTE, a procedure that takes a gexp." (if (gexp? gexp) (delete-duplicates - (append (gexp-self-modules gexp) + (append (self-attribute gexp) (append-map (match-lambda (($ (? gexp? exp)) - (gexp-modules exp)) + (gexp-attribute exp self-attribute)) (($ (lst ...)) (append-map (lambda (item) (if (gexp? item) - (gexp-modules item) + (gexp-attribute item + self-attribute) '())) lst)) (_ @@ -512,6 +515,17 @@ false, meaning that GEXP is a plain Scheme object, return the empty list." (gexp-references gexp)))) '())) ;plain Scheme data type +(define (gexp-modules gexp) + "Return the list of Guile module names GEXP relies on. If (gexp? GEXP) is +false, meaning that GEXP is a plain Scheme object, return the empty list." + (gexp-attribute gexp gexp-self-modules)) + +(define (gexp-extensions gexp) + "Return the list of Guile extensions (packages) GEXP relies on. If (gexp? +GEXP) is false, meaning that GEXP is a plain Scheme object, return the empty +list." + (gexp-attribute gexp gexp-self-extensions)) + (define* (lower-inputs inputs #:key system target) "Turn any package from INPUTS into a derivation for SYSTEM; return the @@ -577,6 +591,7 @@ names and file names suitable for the #:allowed-references argument to (modules '()) (module-path %load-path) (guile-for-build (%guile-for-build)) + (effective-version "2.2") (graft? (%graft?)) references-graphs allowed-references disallowed-references @@ -595,6 +610,9 @@ names of Guile modules searched in MODULE-PATH to be copied in the store, compiled, and made available in the load path during the execution of EXP---e.g., '((guix build utils) (guix build gnu-build-system)). +EFFECTIVE-VERSION determines the string to use when adding extensions of +EXP (see 'with-extensions') to the search path---e.g., \"2.2\". + GRAFT? determines whether packages referred to by EXP should be grafted when applicable. @@ -630,7 +648,7 @@ The other arguments are as for 'derivation'." (define (graphs-file-names graphs) ;; Return a list of (FILE-NAME . STORE-PATH) pairs made from GRAPHS. (map (match-lambda - ;; TODO: Remove 'derivation?' special cases. + ;; TODO: Remove 'derivation?' special cases. ((file-name (? derivation? drv)) (cons file-name (derivation->output-path drv))) ((file-name (? derivation? drv) sub-drv) @@ -639,7 +657,13 @@ The other arguments are as for 'derivation'." (cons file-name thing))) graphs)) - (mlet* %store-monad (;; The following binding forces '%current-system' and + (define (extension-flags extension) + `("-L" ,(string-append (derivation->output-path extension) + "/share/guile/site/" effective-version) + "-C" ,(string-append (derivation->output-path extension) + "/lib/guile/" effective-version "/site-ccache"))) + + (mlet* %store-monad ( ;; The following binding forces '%current-system' and ;; '%current-target-system' to be looked up at >>= ;; time. (graft? (set-grafting graft?)) @@ -660,6 +684,11 @@ The other arguments are as for 'derivation'." #:target target)) (builder (text-file script-name (object->string sexp))) + (extensions -> (gexp-extensions exp)) + (exts (mapm %store-monad + (lambda (obj) + (lower-object obj system)) + extensions)) (modules (if (pair? %modules) (imported-modules %modules #:system system @@ -672,6 +701,7 @@ The other arguments are as for 'derivation'." (compiled-modules %modules #:system system #:module-path module-path + #:extensions extensions #:guile guile-for-build #:deprecation-warnings deprecation-warnings) @@ -704,6 +734,7 @@ The other arguments are as for 'derivation'." `("-L" ,(derivation->output-path modules) "-C" ,(derivation->output-path compiled)) '()) + ,@(append-map extension-flags exts) ,builder) #:outputs outputs #:env-vars env-vars @@ -713,6 +744,7 @@ The other arguments are as for 'derivation'." ,@(if modules `((,modules) (,compiled) ,@inputs) inputs) + ,@(map list exts) ,@(match graphs (((_ . inputs) ...) inputs) (_ '()))) @@ -861,6 +893,17 @@ environment." (identifier-syntax modules))) body ...)) +(define-syntax-parameter current-imported-extensions + ;; Current list of extensions. + (identifier-syntax '())) + +(define-syntax-rule (with-extensions extensions body ...) + "Mark the gexps defined in BODY... as requiring EXTENSIONS in their +execution environment." + (syntax-parameterize ((current-imported-extensions + (identifier-syntax extensions))) + body ...)) + (define-syntax gexp (lambda (s) (define (collect-escapes exp) @@ -957,6 +1000,7 @@ environment." (refs (map escape->ref escapes))) #`(make-gexp (list #,@refs) current-imported-modules + current-imported-extensions (lambda #,formals #,sexp))))))) @@ -1071,6 +1115,7 @@ last one is created from the given object." (system (%current-system)) (guile (%guile-for-build)) (module-path %load-path) + (extensions '()) (deprecation-warnings #f)) "Return a derivation that builds a tree containing the `.go' files corresponding to MODULES. All the MODULES are built in a context where @@ -1129,6 +1174,26 @@ they can refer to each other." (@ (guix build utils) mkdir-p)))) '())) + ;; Add EXTENSIONS to the search path. + ;; TODO: Remove the outer 'ungexp-splicing' on the next rebuild cycle. + (ungexp-splicing + (if (null? extensions) + '() + (gexp ((set! %load-path + (append (map (lambda (extension) + (string-append extension + "/share/guile/site/" + (effective-version))) + '((ungexp-native-splicing extensions))) + %load-path)) + (set! %load-compiled-path + (append (map (lambda (extension) + (string-append extension "/lib/guile/" + (effective-version) + "/site-ccache")) + '((ungexp-native-splicing extensions))) + %load-compiled-path)))))) + (set! %load-path (cons (ungexp modules) %load-path)) (ungexp-splicing @@ -1174,20 +1239,34 @@ they can refer to each other." (module-ref (resolve-interface '(gnu packages guile)) 'guile-2.2)) -(define* (load-path-expression modules #:optional (path %load-path)) +(define* (load-path-expression modules #:optional (path %load-path) + #:key (extensions '())) "Return as a monadic value a gexp that sets '%load-path' and '%load-compiled-path' to point to MODULES, a list of module names. MODULES are searched for in PATH." (mlet %store-monad ((modules (imported-modules modules #:module-path path)) (compiled (compiled-modules modules + #:extensions extensions #:module-path path))) (return (gexp (eval-when (expand load eval) (set! %load-path - (cons (ungexp modules) %load-path)) + (cons (ungexp modules) + (append (map (lambda (extension) + (string-append extension + "/share/guile/site/" + (effective-version))) + '((ungexp-native-splicing extensions))) + %load-path))) (set! %load-compiled-path (cons (ungexp compiled) - %load-compiled-path))))))) + (append (map (lambda (extension) + (string-append extension + "/lib/guile/" + (effective-version) + "/site-ccache")) + '((ungexp-native-splicing extensions))) + %load-compiled-path)))))))) (define* (gexp->script name exp #:key (guile (default-guile)) @@ -1196,7 +1275,9 @@ are searched for in PATH." imported modules in its search path. Look up EXP's modules in MODULE-PATH." (mlet %store-monad ((set-load-path (load-path-expression (gexp-modules exp) - module-path))) + module-path + #:extensions + (gexp-extensions exp)))) (gexp->derivation name (gexp (call-with-output-file (ungexp output) @@ -1225,35 +1306,38 @@ the resulting file. When SET-LOAD-PATH? is true, emit code in the resulting file to set '%load-path' and '%load-compiled-path' to honor EXP's imported modules. Lookup EXP's modules in MODULE-PATH." - (match (if set-load-path? (gexp-modules exp) '()) - (() ;zero modules - (gexp->derivation name - (gexp - (call-with-output-file (ungexp output) - (lambda (port) - (for-each (lambda (exp) - (write exp port)) - '(ungexp (if splice? - exp - (gexp ((ungexp exp))))))))) - #:local-build? #t - #:substitutable? #f)) - ((modules ...) - (mlet %store-monad ((set-load-path (load-path-expression modules - module-path))) - (gexp->derivation name - (gexp - (call-with-output-file (ungexp output) - (lambda (port) - (write '(ungexp set-load-path) port) - (for-each (lambda (exp) - (write exp port)) - '(ungexp (if splice? - exp - (gexp ((ungexp exp))))))))) - #:module-path module-path - #:local-build? #t - #:substitutable? #f))))) + (define modules (gexp-modules exp)) + (define extensions (gexp-extensions exp)) + + (if (or (not set-load-path?) + (and (null? modules) (null? extensions))) + (gexp->derivation name + (gexp + (call-with-output-file (ungexp output) + (lambda (port) + (for-each (lambda (exp) + (write exp port)) + '(ungexp (if splice? + exp + (gexp ((ungexp exp))))))))) + #:local-build? #t + #:substitutable? #f) + (mlet %store-monad ((set-load-path + (load-path-expression modules module-path + #:extensions extensions))) + (gexp->derivation name + (gexp + (call-with-output-file (ungexp output) + (lambda (port) + (write '(ungexp set-load-path) port) + (for-each (lambda (exp) + (write exp port)) + '(ungexp (if splice? + exp + (gexp ((ungexp exp))))))))) + #:module-path module-path + #:local-build? #t + #:substitutable? #f)))) (define* (text-file* name #:rest text) "Return as a monadic value a derivation that builds a text file containing diff --git a/tests/gexp.scm b/tests/gexp.scm index 3c8b4624d..a560adfc5 100644 --- a/tests/gexp.scm +++ b/tests/gexp.scm @@ -23,6 +23,7 @@ #:use-module (guix grafts) #:use-module (guix derivations) #:use-module (guix packages) + #:use-module (guix build-system trivial) #:use-module (guix tests) #:use-module ((guix build utils) #:select (with-directory-excursion)) #:use-module ((guix utils) #:select (call-with-temporary-directory)) @@ -66,6 +67,27 @@ (run-with-store %store exp #:guile-for-build (%guile-for-build)))) +(define %extension-package + ;; Example of a package to use when testing 'with-extensions'. + (dummy-package "extension" + (build-system trivial-build-system) + (arguments + `(#:guile ,%bootstrap-guile + #:modules ((guix build utils)) + #:builder + (begin + (use-modules (guix build utils)) + (let* ((out (string-append (assoc-ref %outputs "out") + "/share/guile/site/" + (effective-version)))) + (mkdir-p out) + (call-with-output-file (string-append out "/hg2g.scm") + (lambda (port) + (write '(define-module (hg2g) + #:export (the-answer)) + port) + (write '(define the-answer 42) port))))))))) + (test-begin "gexp") @@ -739,6 +761,54 @@ (built-derivations (list drv)) (return (= 42 (call-with-input-file out read)))))) +(test-equal "gexp-extensions & ungexp" + (list sed grep) + ((@@ (guix gexp) gexp-extensions) + #~(foo #$(with-extensions (list grep) #~+) + #+(with-extensions (list sed) #~-)))) + +(test-equal "gexp-extensions & ungexp-splicing" + (list grep sed) + ((@@ (guix gexp) gexp-extensions) + #~(foo #$@(list (with-extensions (list grep) #~+) + (with-imported-modules '((foo)) + (with-extensions (list sed) #~-)))))) + +(test-equal "gexp-extensions and literal Scheme object" + '() + ((@@ (guix gexp) gexp-extensions) #t)) + +(test-assertm "gexp->derivation & with-extensions" + ;; Create a fake Guile extension and make sure it is accessible both to the + ;; imported modules and to the derivation build script. + (mlet* %store-monad + ((extension -> %extension-package) + (module -> (scheme-file "x" #~( ;; splice! + (define-module (foo) + #:use-module (hg2g) + #:export (multiply)) + + (define (multiply x) + (* the-answer x))) + #:splice? #t)) + (build -> (with-extensions (list extension) + (with-imported-modules `((guix build utils) + ((foo) => ,module)) + #~(begin + (use-modules (guix build utils) + (hg2g) (foo)) + (call-with-output-file #$output + (lambda (port) + (write (list the-answer (multiply 2)) + port))))))) + (drv (gexp->derivation "thingie" build + ;; %BOOTSTRAP-GUILE is 2.0. + #:effective-version "2.0")) + (out -> (derivation->output-path drv))) + (mbegin %store-monad + (built-derivations (list drv)) + (return (equal? '(42 84) (call-with-input-file out read)))))) + (test-assertm "gexp->derivation #:references-graphs" (mlet* %store-monad ((one (text-file "one" (random-text))) @@ -948,6 +1018,22 @@ (return (and (zero? (close-pipe pipe)) (string=? text str)))))))))) +(test-assertm "program-file & with-extensions" + (let* ((exp (with-extensions (list %extension-package) + (gexp (begin + (use-modules (hg2g)) + (display the-answer))))) + (file (program-file "program" exp + #:guile %bootstrap-guile))) + (mlet* %store-monad ((drv (lower-object file)) + (out -> (derivation->output-path drv))) + (mbegin %store-monad + (built-derivations (list drv)) + (let* ((pipe (open-input-pipe out)) + (str (get-string-all pipe))) + (return (and (zero? (close-pipe pipe)) + (= 42 (string->number str))))))))) + (test-assertm "scheme-file" (let* ((text (plain-file "foo" "Hello, world!")) (scheme (scheme-file "bar" #~(list "foo" #$text)))) -- cgit v1.3 From 7f9d184d9b688d13ce76eefabaddcfa76bdde2b5 Mon Sep 17 00:00:00 2001 From: Caleb Ristvedt Date: Sun, 27 May 2018 19:19:30 +0200 Subject: Add (gnu store database). MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * guix/config.scm.in (%store-database-directory): New variable. * guix/store/database.scm: New file. * tests/store-database.scm: New file. * Makefile.am (STORE_MODULES): New variable. (MODULES, MODULES_NOT_COMPILED): Adjust accordingly. (SCM_TESTS) [HAVE_GUILE_SQLITE3]: Add tests/store-database.scm. Co-authored-by: Ludovic Courtès --- .dir-locals.el | 2 + Makefile.am | 17 ++++ guix/config.scm.in | 6 ++ guix/store/database.scm | 234 +++++++++++++++++++++++++++++++++++++++++++++++ tests/store-database.scm | 54 +++++++++++ 5 files changed, 313 insertions(+) create mode 100644 guix/store/database.scm create mode 100644 tests/store-database.scm (limited to 'tests') diff --git a/.dir-locals.el b/.dir-locals.el index 2db751ca2..eb99a5bcc 100644 --- a/.dir-locals.el +++ b/.dir-locals.el @@ -75,6 +75,8 @@ (eval . (put 'with-imported-modules 'scheme-indent-function 1)) (eval . (put 'with-extensions 'scheme-indent-function 1)) + (eval . (put 'with-database 'scheme-indent-function 2)) + (eval . (put 'call-with-container 'scheme-indent-function 1)) (eval . (put 'container-excursion 'scheme-indent-function 1)) (eval . (put 'eventually 'scheme-indent-function 1)) diff --git a/Makefile.am b/Makefile.am index 2a0a85842..d81fce558 100644 --- a/Makefile.am +++ b/Makefile.am @@ -257,6 +257,16 @@ MODULES += \ endif BUILD_DAEMON_OFFLOAD +# Scheme implementation of the build daemon and related functionality. +STORE_MODULES = \ + guix/store/database.scm + +if HAVE_GUILE_SQLITE3 +MODULES += $(STORE_MODULES) +else +MODULES_NOT_COMPILED += $(STORE_MODULES) +endif !HAVE_GUILE_SQLITE3 + # Internal modules with test suite support. dist_noinst_DATA = guix/tests.scm guix/tests/http.scm @@ -379,6 +389,13 @@ SCM_TESTS += \ endif +if HAVE_GUILE_SQLITE3 + +SCM_TESTS += \ + tests/store-database.scm + +endif + SH_TESTS = \ tests/guix-build.sh \ tests/guix-download.sh \ diff --git a/guix/config.scm.in b/guix/config.scm.in index 8f2c4abd8..dfe5fe0db 100644 --- a/guix/config.scm.in +++ b/guix/config.scm.in @@ -1,5 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2012, 2013, 2014, 2015, 2016 Ludovic Courtès +;;; Copyright © 2017 Caleb Ristvedt ;;; ;;; This file is part of GNU Guix. ;;; @@ -29,6 +30,7 @@ %store-directory %state-directory + %store-database-directory %config-directory %guix-register-program @@ -80,6 +82,10 @@ (or (getenv "NIX_STATE_DIR") (string-append %localstatedir "/guix"))) +(define %store-database-directory + (or (and=> (getenv "NIX_DB_DIR") canonicalize-path) + (string-append %state-directory "/db"))) + (define %config-directory ;; This must match `GUIX_CONFIGURATION_DIRECTORY' as defined in `nix/local.mk'. (or (getenv "GUIX_CONFIGURATION_DIRECTORY") diff --git a/guix/store/database.scm b/guix/store/database.scm new file mode 100644 index 000000000..4233219ba --- /dev/null +++ b/guix/store/database.scm @@ -0,0 +1,234 @@ +;;; GNU Guix --- Functional package management for GNU +;;; Copyright © 2017 Caleb Ristvedt +;;; Copyright © 2018 Ludovic Courtès +;;; +;;; This file is part of GNU Guix. +;;; +;;; GNU Guix is free software; you can redistribute it and/or modify it +;;; under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3 of the License, or (at +;;; your option) any later version. +;;; +;;; GNU Guix is distributed in the hope that it will be useful, but +;;; WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with GNU Guix. If not, see . + +(define-module (guix store database) + #:use-module (sqlite3) + #:use-module (guix config) + #:use-module (guix serialization) + #:use-module (guix base16) + #:use-module (guix hash) + #:use-module (rnrs io ports) + #:use-module (srfi srfi-11) + #:use-module (srfi srfi-19) + #:use-module (ice-9 match) + #:export (sqlite-register + register-path)) + +;;; Code for working with the store database directly. + + +(define-syntax-rule (with-database file db exp ...) + "Open DB from FILE and close it when the dynamic extent of EXP... is left." + (let ((db (sqlite-open file))) + (dynamic-wind noop + (lambda () + exp ...) + (lambda () + (sqlite-close db))))) + +(define (last-insert-row-id db) + ;; XXX: (sqlite3) currently lacks bindings for 'sqlite3_last_insert_rowid'. + ;; Work around that. + (let* ((stmt (sqlite-prepare db "SELECT last_insert_rowid();" + #:cache? #t)) + (result (sqlite-fold cons '() stmt))) + (sqlite-finalize stmt) + (match result + ((#(id)) id) + (_ #f)))) + +(define path-id-sql + "SELECT id FROM ValidPaths WHERE path = :path") + +(define* (path-id db path) + "If PATH exists in the 'ValidPaths' table, return its numerical +identifier. Otherwise, return #f." + (let ((stmt (sqlite-prepare db path-id-sql #:cache? #t))) + (sqlite-bind-arguments stmt #:path path) + (let ((result (sqlite-fold cons '() stmt))) + (sqlite-finalize stmt) + (match result + ((#(id) . _) id) + (_ #f))))) + +(define update-sql + "UPDATE ValidPaths SET hash = :hash, registrationTime = :time, deriver = +:deriver, narSize = :size WHERE id = :id") + +(define insert-sql + "INSERT INTO ValidPaths (path, hash, registrationTime, deriver, narSize) +VALUES (:path, :hash, :time, :deriver, :size)") + +(define* (update-or-insert db #:key path deriver hash nar-size time) + "The classic update-if-exists and insert-if-doesn't feature that sqlite +doesn't exactly have... they've got something close, but it involves deleting +and re-inserting instead of updating, which causes problems with foreign keys, +of course. Returns the row id of the row that was modified or inserted." + (let ((id (path-id db path))) + (if id + (let ((stmt (sqlite-prepare db update-sql #:cache? #t))) + (sqlite-bind-arguments stmt #:id id + #:path path #:deriver deriver + #:hash hash #:size nar-size #:time time) + (sqlite-fold cons '() stmt) + (sqlite-finalize stmt) + (last-insert-row-id db)) + (let ((stmt (sqlite-prepare db insert-sql #:cache? #t))) + (sqlite-bind-arguments stmt + #:path path #:deriver deriver + #:hash hash #:size nar-size #:time time) + (sqlite-fold cons '() stmt) ;execute it + (sqlite-finalize stmt) + (last-insert-row-id db))))) + +(define add-reference-sql + "INSERT OR IGNORE INTO Refs (referrer, reference) SELECT :referrer, id +FROM ValidPaths WHERE path = :reference") + +(define (add-references db referrer references) + "REFERRER is the id of the referring store item, REFERENCES is a list +containing store items being referred to. Note that all of the store items in +REFERENCES must already be registered." + (let ((stmt (sqlite-prepare db add-reference-sql #:cache? #t))) + (for-each (lambda (reference) + (sqlite-reset stmt) + (sqlite-bind-arguments stmt #:referrer referrer + #:reference reference) + (sqlite-fold cons '() stmt) ;execute it + (sqlite-finalize stmt) + (last-insert-row-id db)) + references))) + +;; XXX figure out caching of statement and database objects... later +(define* (sqlite-register #:key db-file path (references '()) + deriver hash nar-size) + "Registers this stuff in a database specified by DB-FILE. PATH is the string +path of some store item, REFERENCES is a list of string paths which the store +item PATH refers to (they need to be already registered!), DERIVER is a string +path of the derivation that created the store item PATH, HASH is the +base16-encoded sha256 hash of the store item denoted by PATH (prefixed with +\"sha256:\") after being converted to nar form, and nar-size is the size in +bytes of the store item denoted by PATH after being converted to nar form." + (with-database db-file db + (let ((id (update-or-insert db #:path path + #:deriver deriver + #:hash hash + #:nar-size nar-size + #:time (time-second (current-time time-utc))))) + (add-references db id references)))) + + +;;; +;;; High-level interface. +;;; + +;; XXX: Would it be better to just make WRITE-FILE give size as well? I question +;; the general utility of this approach. +(define (counting-wrapper-port output-port) + "Some custom ports don't implement GET-POSITION at all. But if we want to +figure out how many bytes are being written, we will want to use that. So this +makes a wrapper around a port which implements GET-POSITION." + (let ((byte-count 0)) + (make-custom-binary-output-port "counting-wrapper" + (lambda (bytes offset count) + (set! byte-count + (+ byte-count count)) + (put-bytevector output-port bytes + offset count) + count) + (lambda () + byte-count) + #f + (lambda () + (close-port output-port))))) + + +(define (nar-sha256 file) + "Gives the sha256 hash of a file and the size of the file in nar form." + (let-values (((port get-hash) (open-sha256-port))) + (let ((wrapper (counting-wrapper-port port))) + (write-file file wrapper) + (force-output wrapper) + (force-output port) + (let ((hash (get-hash)) + (size (port-position wrapper))) + (close-port wrapper) + (values hash size))))) + +;; TODO: make this canonicalize store items that are registered. This involves +;; setting permissions and timestamps, I think. Also, run a "deduplication +;; pass", whatever that involves. Also, handle databases not existing yet +;; (what should the default behavior be? Figuring out how the C++ stuff +;; currently does it sounds like a lot of grepping for global +;; variables...). Also, return #t on success like the documentation says we +;; should. + +(define* (register-path path + #:key (references '()) deriver prefix + state-directory) + ;; Priority for options: first what is given, then environment variables, + ;; then defaults. %state-directory, %store-directory, and + ;; %store-database-directory already handle the "environment variables / + ;; defaults" question, so we only need to choose between what is given and + ;; those. + "Register PATH as a valid store file, with REFERENCES as its list of +references, and DERIVER as its deriver (.drv that led to it.) If PREFIX is +given, it must be the name of the directory containing the new store to +initialize; if STATE-DIRECTORY is given, it must be a string containing the +absolute file name to the state directory of the store being initialized. +Return #t on success. + +Use with care as it directly modifies the store! This is primarily meant to +be used internally by the daemon's build hook." + (let* ((db-dir (cond + (state-directory + (string-append state-directory "/db")) + (prefix + ;; If prefix is specified, the value of NIX_STATE_DIR + ;; (which affects %state-directory) isn't supposed to + ;; affect db-dir, only the compile-time-customized + ;; default should. + (string-append prefix %localstatedir "/guix/db")) + (else + %store-database-directory))) + (store-dir (if prefix + ;; same situation as above + (string-append prefix %storedir) + %store-directory)) + (to-register (if prefix + (string-append %storedir "/" (basename path)) + ;; note: we assume here that if path is, for + ;; example, /foo/bar/gnu/store/thing.txt and prefix + ;; isn't given, then an environment variable has + ;; been used to change the store directory to + ;; /foo/bar/gnu/store, since otherwise real-path + ;; would end up being /gnu/store/thing.txt, which is + ;; probably not the right file in this case. + path)) + (real-path (string-append store-dir "/" (basename path)))) + (let-values (((hash nar-size) + (nar-sha256 real-path))) + (sqlite-register + #:db-file (string-append db-dir "/db.sqlite") + #:path to-register + #:references references + #:deriver deriver + #:hash (string-append "sha256:" + (bytevector->base16-string hash)) + #:nar-size nar-size)))) diff --git a/tests/store-database.scm b/tests/store-database.scm new file mode 100644 index 000000000..1348a75c2 --- /dev/null +++ b/tests/store-database.scm @@ -0,0 +1,54 @@ +;;; GNU Guix --- Functional package management for GNU +;;; Copyright © 2017, 2018 Ludovic Courtès +;;; +;;; This file is part of GNU Guix. +;;; +;;; GNU Guix is free software; you can redistribute it and/or modify it +;;; under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3 of the License, or (at +;;; your option) any later version. +;;; +;;; GNU Guix is distributed in the hope that it will be useful, but +;;; WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with GNU Guix. If not, see . + +(define-module (test-store-database) + #:use-module (guix tests) + #:use-module ((guix store) #:hide (register-path)) + #:use-module (guix store database) + #:use-module (srfi srfi-26) + #:use-module (srfi srfi-64)) + +;; Test the (guix store database) module. + +(define %store + (open-connection-for-tests)) + + +(test-begin "store-database") + +(test-assert "register-path" + (let ((file (string-append (%store-prefix) "/" (make-string 32 #\f) + "-fake"))) + (when (valid-path? %store file) + (delete-paths %store (list file))) + (false-if-exception (delete-file file)) + + (let ((ref (add-text-to-store %store "ref-of-fake" (random-text))) + (drv (string-append file ".drv"))) + (call-with-output-file file + (cut display "This is a fake store item.\n" <>)) + (register-path file + #:references (list ref) + #:deriver drv) + + (and (valid-path? %store file) + (equal? (references %store file) (list ref)) + (null? (valid-derivers %store file)) + (null? (referrers %store file)))))) + +(test-end "store-database") -- cgit v1.3 From bf5bf5778cb7c3a2475c6acd707abc925b1819aa Mon Sep 17 00:00:00 2001 From: Caleb Ristvedt Date: Sun, 27 May 2018 23:20:54 +0200 Subject: Add (guix store deduplication). MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * guix/store/database.scm (register-path): Add #:deduplicate? and call 'deduplicate' when it's true. (counting-wrapper-port, nar-sha256): Move to... * guix/store/deduplication.scm: ... here. New file. * tests/store-deduplication.scm: New file. * Makefile.am (STORE_MODULES): Add deduplication.scm. (SCM_TESTS) [HAVE_GUILE_SQLITE3]: Add store-deduplication.scm. Co-authored-by: Ludovic Courtès --- Makefile.am | 6 +- guix/store/database.scm | 43 ++---------- guix/store/deduplication.scm | 148 ++++++++++++++++++++++++++++++++++++++++++ tests/store-deduplication.scm | 64 ++++++++++++++++++ 4 files changed, 222 insertions(+), 39 deletions(-) create mode 100644 guix/store/deduplication.scm create mode 100644 tests/store-deduplication.scm (limited to 'tests') diff --git a/Makefile.am b/Makefile.am index d81fce558..474575c9f 100644 --- a/Makefile.am +++ b/Makefile.am @@ -259,7 +259,8 @@ endif BUILD_DAEMON_OFFLOAD # Scheme implementation of the build daemon and related functionality. STORE_MODULES = \ - guix/store/database.scm + guix/store/database.scm \ + guix/store/deduplication.scm if HAVE_GUILE_SQLITE3 MODULES += $(STORE_MODULES) @@ -392,7 +393,8 @@ endif if HAVE_GUILE_SQLITE3 SCM_TESTS += \ - tests/store-database.scm + tests/store-database.scm \ + tests/store-deduplication.scm endif diff --git a/guix/store/database.scm b/guix/store/database.scm index b9745dbe1..3623c0e7a 100644 --- a/guix/store/database.scm +++ b/guix/store/database.scm @@ -21,10 +21,9 @@ #:use-module (sqlite3) #:use-module (guix config) #:use-module (guix serialization) + #:use-module (guix store deduplication) #:use-module (guix base16) - #:use-module (guix hash) #:use-module (guix build syscalls) - #:use-module (rnrs io ports) #:use-module (srfi srfi-11) #:use-module (srfi srfi-19) #:use-module (ice-9 match) @@ -140,39 +139,6 @@ bytes of the store item denoted by PATH after being converted to nar form." ;;; High-level interface. ;;; -;; XXX: Would it be better to just make WRITE-FILE give size as well? I question -;; the general utility of this approach. -(define (counting-wrapper-port output-port) - "Some custom ports don't implement GET-POSITION at all. But if we want to -figure out how many bytes are being written, we will want to use that. So this -makes a wrapper around a port which implements GET-POSITION." - (let ((byte-count 0)) - (make-custom-binary-output-port "counting-wrapper" - (lambda (bytes offset count) - (set! byte-count - (+ byte-count count)) - (put-bytevector output-port bytes - offset count) - count) - (lambda () - byte-count) - #f - (lambda () - (close-port output-port))))) - - -(define (nar-sha256 file) - "Gives the sha256 hash of a file and the size of the file in nar form." - (let-values (((port get-hash) (open-sha256-port))) - (let ((wrapper (counting-wrapper-port port))) - (write-file file wrapper) - (force-output wrapper) - (force-output port) - (let ((hash (get-hash)) - (size (port-position wrapper))) - (close-port wrapper) - (values hash size))))) - ;; TODO: Factorize with that in (gnu build install). (define (reset-timestamps file) "Reset the modification time on FILE and on all the files it contains, if @@ -211,7 +177,7 @@ it's a directory." (define* (register-path path #:key (references '()) deriver prefix - state-directory) + state-directory (deduplicate? #t)) ;; Priority for options: first what is given, then environment variables, ;; then defaults. %state-directory, %store-directory, and ;; %store-database-directory already handle the "environment variables / @@ -262,4 +228,7 @@ be used internally by the daemon's build hook." #:deriver deriver #:hash (string-append "sha256:" (bytevector->base16-string hash)) - #:nar-size nar-size)))) + #:nar-size nar-size) + + (when deduplicate? + (deduplicate real-path hash #:store store-dir))))) diff --git a/guix/store/deduplication.scm b/guix/store/deduplication.scm new file mode 100644 index 000000000..4b4ac01f6 --- /dev/null +++ b/guix/store/deduplication.scm @@ -0,0 +1,148 @@ +;;; GNU Guix --- Functional package management for GNU +;;; Copyright © 2017 Caleb Ristvedt +;;; Copyright © 2018 Ludovic Courtès +;;; +;;; This file is part of GNU Guix. +;;; +;;; GNU Guix is free software; you can redistribute it and/or modify it +;;; under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3 of the License, or (at +;;; your option) any later version. +;;; +;;; GNU Guix is distributed in the hope that it will be useful, but +;;; WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with GNU Guix. If not, see . + +;;; This houses stuff we do to files when they arrive at the store - resetting +;;; timestamps, deduplicating, etc. + +(define-module (guix store deduplication) + #:use-module (guix hash) + #:use-module (guix build utils) + #:use-module (guix base16) + #:use-module (srfi srfi-11) + #:use-module (rnrs io ports) + #:use-module (ice-9 ftw) + #:use-module (guix serialization) + #:export (nar-sha256 + deduplicate)) + +;; Would it be better to just make WRITE-FILE give size as well? I question +;; the general utility of this approach. +(define (counting-wrapper-port output-port) + "Some custom ports don't implement GET-POSITION at all. But if we want to +figure out how many bytes are being written, we will want to use that. So this +makes a wrapper around a port which implements GET-POSITION." + (let ((byte-count 0)) + (make-custom-binary-output-port "counting-wrapper" + (lambda (bytes offset count) + (set! byte-count + (+ byte-count count)) + (put-bytevector output-port bytes + offset count) + count) + (lambda () + byte-count) + #f + (lambda () + (close-port output-port))))) + +(define (nar-sha256 file) + "Gives the sha256 hash of a file and the size of the file in nar form." + (let-values (((port get-hash) (open-sha256-port))) + (let ((wrapper (counting-wrapper-port port))) + (write-file file wrapper) + (force-output wrapper) + (force-output port) + (let ((hash (get-hash)) + (size (port-position wrapper))) + (close-port wrapper) + (values hash size))))) + +(define (tempname-in directory) + "Gives an unused temporary name under DIRECTORY. Not guaranteed to still be +unused by the time you create anything with that name, but a good shot." + (let ((const-part (string-append directory "/.tmp-link-" + (number->string (getpid))))) + (let try ((guess-part + (number->string (random most-positive-fixnum) 16))) + (if (file-exists? (string-append const-part "-" guess-part)) + (try (number->string (random most-positive-fixnum) 16)) + (string-append const-part "-" guess-part))))) + +(define* (get-temp-link target #:optional (link-prefix (dirname target))) + "Like mkstemp!, but instead of creating a new file and giving you the name, +it creates a new hardlink to TARGET and gives you the name. Since +cross-filesystem hardlinks don't work, the temp link must be created on the +same filesystem - where in that filesystem it is can be controlled by +LINK-PREFIX." + (let try ((tempname (tempname-in link-prefix))) + (catch 'system-error + (lambda () + (link target tempname) + tempname) + (lambda (args) + (if (= (system-error-errno args) EEXIST) + (try (tempname-in link-prefix)) + (throw 'system-error args)))))) + +;; There are 3 main kinds of errors we can get from hardlinking: "Too many +;; things link to this" (EMLINK), "this link already exists" (EEXIST), and +;; "can't fit more stuff in this directory" (ENOSPC). + +(define (replace-with-link target to-replace) + "Atomically replace the file TO-REPLACE with a link to TARGET. Note: TARGET +and TO-REPLACE must be on the same file system." + (let ((temp-link (get-temp-link target (dirname to-replace)))) + (rename-file temp-link to-replace))) + +(define-syntax-rule (false-if-system-error (errors ...) exp ...) + "Given ERRORS, a list of system error codes to ignore, evaluates EXP... and +return #f if any of the system error codes in the given list are thrown." + (catch 'system-error + (lambda () + exp ...) + (lambda args + (if (member (system-error-errno args) (list errors ...)) + #f + (apply throw args))))) + +(define* (deduplicate path hash #:key (store %store-directory)) + "Check if a store item with sha256 hash HASH already exists. If so, +replace PATH with a hardlink to the already-existing one. If not, register +PATH so that future duplicates can hardlink to it. PATH is assumed to be +under STORE." + (let* ((links-directory (string-append store "/.links")) + (link-file (string-append links-directory "/" + (bytevector->base16-string hash)))) + (mkdir-p links-directory) + (if (file-is-directory? path) + ;; Can't hardlink directories, so hardlink their atoms. + (for-each (lambda (file) + (unless (member file '("." "..")) + (deduplicate file (nar-sha256 file) + #:store store))) + (scandir path)) + (if (file-exists? link-file) + (false-if-system-error (EMLINK) + (replace-with-link link-file path)) + (catch 'system-error + (lambda () + (link path link-file)) + (lambda args + (let ((errno (system-error-errno args))) + (cond ((= errno EEXIST) + ;; Someone else put an entry for PATH in + ;; LINKS-DIRECTORY before we could. Let's use it. + (false-if-system-error (EMLINK) + (replace-with-link path link-file))) + ((= errno ENOSPC) + ;; There's not enough room in the directory index for + ;; more entries in .links, but that's fine: we can + ;; just stop. + #f) + (else (apply throw args)))))))))) diff --git a/tests/store-deduplication.scm b/tests/store-deduplication.scm new file mode 100644 index 000000000..04817a193 --- /dev/null +++ b/tests/store-deduplication.scm @@ -0,0 +1,64 @@ +;;; GNU Guix --- Functional package management for GNU +;;; Copyright © 2018 Ludovic Courtès +;;; +;;; This file is part of GNU Guix. +;;; +;;; GNU Guix is free software; you can redistribute it and/or modify it +;;; under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3 of the License, or (at +;;; your option) any later version. +;;; +;;; GNU Guix is distributed in the hope that it will be useful, but +;;; WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with GNU Guix. If not, see . + +(define-module (test-store-deduplication) + #:use-module (guix tests) + #:use-module (guix store deduplication) + #:use-module (guix hash) + #:use-module ((guix utils) #:select (call-with-temporary-directory)) + #:use-module (guix build utils) + #:use-module (rnrs bytevectors) + #:use-module (ice-9 binary-ports) + #:use-module (srfi srfi-1) + #:use-module (srfi srfi-64)) + +(test-begin "store-deduplication") + +(test-equal "deduplicate" + (cons* #t #f ;inode comparisons + 2 (make-list 5 6)) ;'nlink' values + + (call-with-temporary-directory + (lambda (store) + (let ((data (string->utf8 "Hello, world!")) + (identical (map (lambda (n) + (string-append store "/" (number->string n))) + (iota 5))) + (unique (string-append store "/unique"))) + (for-each (lambda (file) + (call-with-output-file file + (lambda (port) + (put-bytevector port data)))) + identical) + (call-with-output-file unique + (lambda (port) + (put-bytevector port (string->utf8 "This is unique.")))) + + (for-each (lambda (file) + (deduplicate file (sha256 data) #:store store)) + identical) + (deduplicate unique (nar-sha256 unique) #:store store) + + ;; (system (string-append "ls -lRia " store)) + (cons* (apply = (map (compose stat:ino stat) identical)) + (= (stat:ino (stat unique)) + (stat:ino (stat (car identical)))) + (stat:nlink (stat unique)) + (map (compose stat:nlink stat) identical)))))) + +(test-end "store-deduplication") -- cgit v1.3 From a99b0ad785d5b31e9d9efd2d186ab5d53544b631 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Wed, 6 Jun 2018 13:58:43 +0200 Subject: tests: Adjust graph test. This is a followup to 301a4249064227bc4883e4e5b638e85a65925ba0, which added a dependency to sed. * tests/graph.scm ("node-edges"): Use 'hello', not 'sed', as the example of a package without any dependency. --- tests/graph.scm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tests') diff --git a/tests/graph.scm b/tests/graph.scm index b86ae4a32..4799d3bd0 100644 --- a/tests/graph.scm +++ b/tests/graph.scm @@ -293,7 +293,7 @@ edges." (run-with-store %store (let ((packages (fold-packages cons '()))) (mlet %store-monad ((edges (node-edges %package-node-type packages))) - (return (and (null? (edges sed)) + (return (and (null? (edges hello)) (lset= eq? (edges guile-2.0) (match (package-direct-inputs guile-2.0) -- cgit v1.3 From 477ee222aa796257b5f32966da662e76fe706332 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Wed, 6 Jun 2018 14:26:09 +0200 Subject: tests: Adjust 'guix package' test to "python2" name. This is a followup to a7714d42de2c3082f3609d1e63c83d703fb39cf9, which renamed Python 2.x to "python2". * tests/guix-package.sh: Use g-wrap and guile@2.0 when testing collisions. --- tests/guix-package.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'tests') diff --git a/tests/guix-package.sh b/tests/guix-package.sh index aa5eaa66e..3b3fa35cd 100644 --- a/tests/guix-package.sh +++ b/tests/guix-package.sh @@ -60,12 +60,12 @@ test -L "$profile" && test -L "$profile-1-link" ! test -f "$profile-2-link" test -f "$profile/bin/guile" -# Collisions are properly flagged (in this case, 'python-wrapper' propagates -# python@3, which conflicts with python@2.) -if guix package --bootstrap -n -p "$profile" -i python@2 python-wrapper +# Collisions are properly flagged (in this case, 'g-wrap' propagates +# guile@2.2, which conflicts with guile@2.0.) +if guix package --bootstrap -n -p "$profile" -i g-wrap guile@2.0 then false; else true; fi -guix package --bootstrap -n -p "$profile" -i python@2 python-wrapper \ +guix package --bootstrap -n -p "$profile" -i g-wrap guile@2.0 \ --allow-collisions # No search path env. var. here. -- cgit v1.3 From c423ae89185abab9ca6381a12285b85079367072 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Sun, 13 May 2018 18:46:13 +0200 Subject: packages: Add 'package-patched-vulnerabilities'. * guix/packages.scm (patch-file-name): New procedure. (%vulnerability-regexp): New variable. (package-patched-vulnerabilities): New procedure. * guix/scripts/lint.scm (patch-file-name): Remove. (check-vulnerabilities): Adjust to use 'package-patched-vulnerabilities'. * tests/packages.scm ("package-patched-vulnerabilities"): New test. --- guix/packages.scm | 28 ++++++++++++++++++++++++++++ guix/scripts/lint.scm | 23 ++++------------------- tests/packages.scm | 15 +++++++++++++++ 3 files changed, 47 insertions(+), 19 deletions(-) (limited to 'tests') diff --git a/guix/packages.scm b/guix/packages.scm index a6f9936d6..c762fa7c3 100644 --- a/guix/packages.scm +++ b/guix/packages.scm @@ -35,6 +35,7 @@ #:use-module (guix sets) #:use-module (ice-9 match) #:use-module (ice-9 vlist) + #:use-module (ice-9 regex) #:use-module (srfi srfi-1) #:use-module (srfi srfi-9 gnu) #:use-module (srfi srfi-11) @@ -106,6 +107,7 @@ package-cross-derivation package-output package-grafts + package-patched-vulnerabilities package/inherit transitive-input-references @@ -394,6 +396,32 @@ DELIMITER (a string), you can customize what will appear between the name and the version. By default, DELIMITER is \"@\"." (string-append (package-name package) delimiter (package-version package))) +(define (patch-file-name patch) + "Return the basename of PATCH's file name, or #f if the file name could not +be determined." + (match patch + ((? string?) + (basename patch)) + ((? origin?) + (and=> (origin-actual-file-name patch) basename)))) + +(define %vulnerability-regexp + ;; Regexp matching a CVE identifier in patch file names. + (make-regexp "CVE-[0-9]{4}-[0-9]+")) + +(define (package-patched-vulnerabilities package) + "Return the list of patched vulnerabilities of PACKAGE as a list of CVE +identifiers. The result is inferred from the file names of patches." + (define (patch-vulnerabilities patch) + (map (cut match:substring <> 0) + (list-matches %vulnerability-regexp patch))) + + (let ((patches (filter-map patch-file-name + (or (and=> (package-source package) + origin-patches) + '())))) + (append-map patch-vulnerabilities patches))) + (define (%standard-patch-inputs) (let* ((canonical (module-ref (resolve-interface '(gnu packages base)) 'canonical-package)) diff --git a/guix/scripts/lint.scm b/guix/scripts/lint.scm index cd802985d..e477bf0dd 100644 --- a/guix/scripts/lint.scm +++ b/guix/scripts/lint.scm @@ -1,7 +1,7 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2014 Cyril Roelandt ;;; Copyright © 2014, 2015 Eric Bavier -;;; Copyright © 2013, 2014, 2015, 2016, 2017 Ludovic Courtès +;;; Copyright © 2013, 2014, 2015, 2016, 2017, 2018 Ludovic Courtès ;;; Copyright © 2015, 2016 Mathieu Lirzin ;;; Copyright © 2016 Danny Milosavljevic ;;; Copyright © 2016 Hartmut Goebel @@ -809,15 +809,6 @@ descriptions maintained upstream." (emit-warning package (G_ "invalid license field") 'license)))) -(define (patch-file-name patch) - "Return the basename of PATCH's file name, or #f if the file name could not -be determined." - (match patch - ((? string?) - (basename patch)) - ((? origin?) - (and=> (origin-actual-file-name patch) basename)))) - (define (call-with-networking-fail-safe message error-value proc) "Call PROC catching any network-related errors. Upon a networking error, display a message including MESSAGE and return ERROR-VALUE." @@ -878,20 +869,14 @@ the NIST server non-fatal." (() #t) ((vulnerabilities ...) - (let* ((patches (filter-map patch-file-name - (or (and=> (package-source package) - origin-patches) - '()))) + (let* ((patched (package-patched-vulnerabilities package)) (known-safe (or (assq-ref (package-properties package) 'lint-hidden-cve) '())) (unpatched (remove (lambda (vuln) (let ((id (vulnerability-id vuln))) - (or - (find (cute string-contains - <> id) - patches) - (member id known-safe)))) + (or (member id patched) + (member id known-safe)))) vulnerabilities))) (unless (null? unpatched) (emit-warning package diff --git a/tests/packages.scm b/tests/packages.scm index f1e7d3119..65ccb1488 100644 --- a/tests/packages.scm +++ b/tests/packages.scm @@ -959,6 +959,21 @@ ((("x" dep)) (eq? dep findutils))))))))) +(test-equal "package-patched-vulnerabilities" + '(("CVE-2015-1234") + ("CVE-2016-1234" "CVE-2018-4567") + ()) + (let ((p1 (dummy-package "pi" + (source (dummy-origin + (patches (list "/a/b/pi-CVE-2015-1234.patch")))))) + (p2 (dummy-package "pi" + (source (dummy-origin + (patches (list + "/a/b/pi-CVE-2016-1234-CVE-2018-4567.patch")))))) + (p3 (dummy-package "pi" (source (dummy-origin))))) + (map package-patched-vulnerabilities + (list p1 p2 p3)))) + (test-eq "fold-packages" hello (fold-packages (lambda (p r) (if (string=? (package-name p) "hello") -- cgit v1.3