code:
;;; GNU Guix --- Functional package management for GNU
;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018 Ludovic Courtès <ludo@gnu.org>
;;; Copyright © 2013 Nikita Karetnikov <nikita@karetnikov.org>
;;; Copyright © 2013, 2015 Mark H Weaver <mhw@netris.org>
;;; Copyright © 2014, 2016 Alex Kost <alezost@gmail.com>
;;; Copyright © 2016 Roel Janssen <roel@gnu.org>
;;; Copyright © 2016 Benz Schenk <benz.schenk@uzh.ch>
;;; Copyright © 2016 Chris Marusich <cmmarusich@gmail.com>
;;;
;;; This file is part of GNU Guix.
;;;
;;; GNU Guix is free software; you can redistribute it and/or modify it
;;; under the terms of the GNU General Public License as published by
;;; the Free Software Foundation; either version 3 of the License, or (at
;;; your option) any later version.
;;;
;;; GNU Guix is distributed in the hope that it will be useful, but
;;; WITHOUT ANY WARRANTY; without even the implied warranty of
;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
;;; GNU General Public License for more details.
;;;
;;; You should have received a copy of the GNU General Public License
;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.

(define-module (guix scripts package)
  #:use-module (guix ui)
  #:use-module (guix status)
  #:use-module (guix store)
  #:use-module (guix grafts)
  #:use-module (guix derivations)
  #:use-module (guix packages)
  #:use-module (guix profiles)
  #:use-module (guix search-paths)
  #:use-module (guix monads)
  #:use-module (guix utils)
  #:use-module (guix config)
  #:use-module (guix scripts)
  #:use-module (guix scripts build)
  #:autoload   (guix describe) (current-profile-entries)
  #:use-module ((guix build utils)
                #:select (directory-exists? mkdir-p))
  #:use-module (ice-9 format)
  #:use-module (ice-9 match)
  #:use-module (ice-9 regex)
  #:use-module (ice-9 vlist)
  #:use-module (srfi srfi-1)
  #:use-module (srfi srfi-11)
  #:use-module (srfi srfi-26)
  #:use-module (srfi srfi-34)
  #:use-module (srfi srfi-35)
  #:use-module (srfi srfi-37)
  #:use-module (gnu packages)
  #:autoload   (gnu packages base) (canonical-package)
  #:autoload   (gnu packages guile) (guile-2.2)
  #:autoload   (gnu packages bootstrap) (%bootstrap-guile)
  #:export (build-and-use-profile
            delete-generations
            display-search-paths
            guix-package))

(define %store
  (make-parameter #f))

;;;
;;; Profiles.
;;;

(define (ensure-default-profile)
  "Ensure the default profile symlink and directory exist and are writable."
  (ensure-profile-directory)

  ;; Create ~/.guix-profile if it doesn't exist yet.
  (when (and %user-profile-directory
             %current-profile
             (not (false-if-exception
                   (lstat %user-profile-directory))))
    (symlink %current-profile %user-profile-directory)))

(define (delete-generations store profile generations)
  "Delete GENERATIONS from PROFILE.
GENERATIONS is a list of generation numbers."
  (for-each (cut delete-generation* store profile <>)
            generations))

(define (delete-matching-generations store profile pattern)
  "Delete from PROFILE all the generations matching PATTERN.  PATTERN must be
a string denoting a set of generations: the empty list means \"all generations
but the current one\", a number designates a generation, and other patterns
denote ranges as interpreted by 'matching-generations'."
  (let ((current (generation-number profile)))
    (cond ((not (file-exists? profile))            ; XXX: race condition
           (raise (condition (&profile-not-found-error
                              (profile profile)))))
          ((string-null? pattern)
           (delete-generations store profile
                               (delv current (profile-generations profile))))
          ;; Do not delete the zeroth generation.
          ((equal? 0 (string->number pattern))
           #t)

          ;; If PATTERN is a duration, match generations that are
          ;; older than the specified duration.
          ((matching-generations pattern profile
                                 #:duration-relation >)
           =>
           (lambda (numbers)
             (when (memv current numbers)
               (warning (G_ "not removing generation ~a, which is current~%")
                        current))

             ;; Make sure we don't inadvertently remove the current
             ;; generation.
             (let ((numbers (delv current numbers)))
               (when (null-list? numbers)
                 (leave (G_ "no matching generation~%")))
               (delete-generations store profile numbers))))
          (else
           (leave (G_ "invalid syntax: ~a~%") pattern)))))

(define* (build-and-use-profile store profile manifest
                                #:key
                                allow-collisions?
                                bootstrap? use-substitutes?
                                dry-run?)
  "Build a new generation of PROFILE, a file name, using the packages
specified in MANIFEST, a manifest object.  When ALLOW-COLLISIONS? is true,
do not treat collisions in MANIFEST as an error."
  (when (equal? profile %current-profile)
    (ensure-default-profile))

  (let* ((prof-drv (run-with-store store
                     (profile-derivation manifest
                                         #:allow-collisions? allow-collisions?
                                         #:hooks (if bootstrap?
                                                     '()
                                                     %default-profile-hooks)
                                         #:locales? (not bootstrap?))))
         (prof     (derivation->output-path prof-drv)))
    (show-what-to-build store (list prof-drv)
                        #:use-substitutes? use-substitutes?
                        #:dry-run? dry-run?)

    (cond
     (dry-run? #t)
     ((and (file-exists? profile)
           (and=> (readlink* profile) (cut string=? prof <>)))
      (format (current-error-port) (G_ "nothing to be done~%")))
     (else
      (let* ((number (generation-number profile))

             ;; Always use NUMBER + 1 for the new profile, possibly
             ;; overwriting a "previous future generation".
             (name   (generation-file-name profile (+ 1 number))))
        (and (build-derivations store (list prof-drv))
             (let* ((entries (manifest-entries manifest))
                    (count   (length entries)))
               (switch-symlinks name prof)
               (switch-symlinks profile (basename name))
               (unless (string=? profile %current-profile)
                 (register-gc-root store name))
               (format #t (N_ "~a package in profile~%"
                              "~a packages in profile~%"
                              count)
                       count)
               (display-search-paths entries (list profile)
                                     #:kind 'prefix))))))))

;;;
;;; Package specifications.
;;;

(define (find-packages-by-description regexps)
  "Return two values: the list of packages whose name, synopsis, or
description matches at least one of REGEXPS sorted by relevance, and the list
of relevance scores."
  (let ((matches (fold-packages (lambda (package result)
                                  (if (package-superseded package)
                                      result
                                      (match (package-relevance package
                                                                regexps)
                                        ((? zero?)
                                         result)
                                        (score
                                         (cons (list package score)
                                               result)))))
                                '())))
    (unzip2 (sort matches
                  (lambda (m1 m2)
                    (match m1
                      ((package1 score1)
                       (match m2
                         ((package2 score2)
                          (if (= score1 score2)
                              (string>? (package-full-name package1)
                                        (package-full-name package2))
                              (> score1 score2)))))))))))

(define (transaction-upgrade-entry entry transaction)
  "Return a variant of TRANSACTION that accounts for the upgrade of ENTRY, a
<manifest-entry>."
  (define (supersede old new)
    (info (G_ "package '~a' has been superseded by '~a'~%")
          (manifest-entry-name old) (package-name new))
    (manifest-transaction-install-entry
     (package->manifest-entry* new (manifest-entry-output old))
     (manifest-transaction-remove-pattern
      (manifest-pattern
        (name (manifest-entry-name old))
        (version (manifest-entry-version old))
        (output (manifest-entry-output old)))
      transaction)))

  (match (if (manifest-transaction-removal-candidate? entry transaction)
             'dismiss
             entry)
    ('dismiss
     transaction)
    (($ <manifest-entry> name version output (? string? path))
     (match (vhash-assoc name (find-newest-available-packages))
       ((_ candidate-version pkg . rest)
        (match (package-superseded pkg)
          ((? package? new)
           (supersede entry new))
          (#f
           (case (version-compare candidate-version version)
             ((>)
              (manifest-transaction-install-entry
               (package->manifest-entry* pkg output)
               transaction))
             ((<)
              transaction)
             ((=)
              (let ((candidate-path (derivation->output-path
                                     (package-derivation (%store) pkg))))
                ;; XXX: When there are propagated inputs, assume we need to
                ;; upgrade the whole entry.
                (if (and (string=? path candidate-path)
                         (null? (package-propagated-inputs pkg)))
                    transaction
                    (manifest-transaction-install-entry
                     (package->manifest-entry* pkg output)
                     transaction))))))))
       (#f
        (warning (G_ "package '~a' no longer exists~%") name)
        transaction)))))

;;;
;;; Search paths.
;;;

(define* (search-path-environment-variables entries profiles
                                            #:optional (getenv getenv)
                                            #:key (kind 'exact))
  "Return environment variable definitions that may be needed for the use of
ENTRIES, a list of manifest entries, in PROFILES.  Use GETENV to determine the
current settings and report only settings not already effective.  KIND
must be one of 'exact, 'prefix, or 'suffix, depending on the kind of search
path definition to be returned."
  (let ((search-paths (delete-duplicates
                       (cons $PATH
                             (append-map manifest-entry-search-paths
                                         entries)))))
    (filter-map (match-lambda
                  ((spec . value)
                   (let ((variable (search-path-specification-variable spec))
                         (sep      (search-path-specification-separator spec)))
                     (environment-variable-definition variable value
                                                      #:separator sep
                                                      #:kind kind))))
                (evaluate-search-paths search-paths profiles
                                       getenv))))

(define* (display-search-paths entries profiles
                               #:key (kind 'exact))
  "Display the search path environment variables that may need to be set for
ENTRIES, a list of manifest entries, in the context of PROFILE."
  (let* ((profiles (map user-friendly-profile profiles))
         (settings (search-path-environment-variables entries profiles
                                                      #:kind kind)))
    (unless (null? settings)
      (format #t (G_ "The following environment variable definitions may be needed:~%"))
      (format #t "~{   ~a~%~}" settings))))

;;;
;;; Command-line options.
;;;

(define %default-options
  ;; Alist of default option values.
  `((verbosity . 0)
    (graft? . #t)
    (substitutes? . #t)
    (build-hook? . #t)
    (print-build-trace? . #t)
    (print-extended-build-trace? . #t)))

(define (show-help)
  (display (G_ "Usage: guix package [OPTION]...
Install, remove, or upgrade packages in a single transaction.\n"))
  (display (G_ "
  -i, --install PACKAGE ...
                         install PACKAGEs"))
  (display (G_ "
  -e, --install-from-expression=EXP
                         install the package EXP evaluates to"))
  (display (G_ "
  -f, --install-from-file=FILE
                         install the package that the code within FILE
                         evaluates to"))
  (display (G_ "
  -r, --remove PACKAGE ...
                         remove PACKAGEs"))
  (display (G_ "
  -u, --upgrade[=REGEXP] upgrade all the installed packages matching REGEXP"))
  (display (G_ "
  -m, --manifest=FILE    create a new profile generation with the manifest
                         from FILE"))
  (display (G_ "
      --do-not-upgrade[=REGEXP] do not upgrade any packages matching REGEXP"))
  (display (G_ "
      --roll-back        roll back to the previous generation"))
  (display (G_ "
      --search-paths[=KIND]
                         display needed environment variable definitions"))
  (display (G_ "
  -l, --list-generations[=PATTERN]
                         list generations matching PATTERN"))
  (display (G_ "
  -d, --delete-generations[=PATTERN]
                         delete generations matching PATTERN"))
  (display (G_ "
  -S, --switch-generation=PATTERN
                         switch to a generation matching PATTERN"))
  (display (G_ "
  -p, --profile=PROFILE  use PROFILE instead of the user's default profile"))
  (newline)
  (display (G_ "
      --allow-collisions do not treat collisions in the profile as an error"))
  (display (G_ "
      --bootstrap        use the bootstrap Guile to build the profile"))
  (display (G_ "
      --verbose          produce verbose output"))
  (newline)
  (display (G_ "
  -s, --search=REGEXP    search in synopsis and description using REGEXP"))
  (display (G_ "
  -I, --list-installed[=REGEXP]
                         list installed packages matching REGEXP"))
  (display (G_ "
  -A, --list-available[=REGEXP]
                         list available packages matching REGEXP"))
  (display (G_ "
      --show=PACKAGE     show details about PACKAGE"))
  (newline)
  (show-build-options-help)
  (newline)
  (show-transformation-options-help)
  (newline)
  (display (G_ "
  -h, --help             display this help and exit"))
  (display (G_ "
  -V, --version          display version information and exit"))
  (newline)
  (show-bug-report-information))

(define %options
  ;; Specification of the command-line options.
  (cons* (option '(#\h "help") #f #f
                 (lambda args
                   (show-help)
                   (exit 0)))
         (option '(#\V "version") #f #f
                 (lambda args
                   (show-version-and-exit "guix package")))

         (option '(#\i "install") #f #t
                 (lambda (opt name arg result arg-handler)
                   (let arg-handler ((arg arg) (result result))
                     (values (if arg
                                 (alist-cons 'install arg result)
                                 result)
                             arg-handler))))
         (option '(#\e "install-from-expression") #t #f
                 (lambda (opt name arg result arg-handler)
                   (values (alist-cons 'install (read/eval-package-expression arg)
                                       result)
                           #f)))
         (option '(#\f "install-from-file") #t #f
                 (lambda (opt name arg result arg-handler)
                   (values (alist-cons 'install
                                       (load* arg (make-user-module '()))
                                       result)
                           #f)))
         (option '(#\r "remove") #f #t
                 (lambda (opt name arg result arg-handler)
                   (let arg-handler ((arg arg) (result result))
                     (values (if arg
                                 (alist-cons 'remove arg result)
                                 result)
                             arg-handler))))
         (option '(#\u "upgrade") #f #t
                 (lambda (opt name arg result arg-handler)
                   (when (and arg (string-prefix? "-" arg))
                     (warning (G_ "upgrade regexp '~a' looks like a \
command-line option~%")
                              arg)
                     (warning (G_ "is this intended?~%")))
                   (let arg-handler ((arg arg) (result result))
                     (values (alist-cons 'upgrade arg
                                         ;; Delete any prior "upgrade all"
                                         ;; command, or else "--upgrade gcc"
                                         ;; would upgrade everything.
                                         (delete '(upgrade . #f) result))
                             arg-handler))))
         (option '("do-not-upgrade") #f #t
                 (lambda (opt name arg result arg-handler)
                   (let arg-handler ((arg arg) (result result))
                     (values (if arg
                                 (alist-cons 'do-not-upgrade arg result)
                                 result)
                             arg-handler))))
         (option '("roll-back" "rollback") #f #f
                 (lambda (opt name arg result arg-handler)
                   (values (alist-cons 'roll-back? #t result)
                           #f)))
         (option '(#\m "manifest") #t #f
                 (lambda (opt name arg result arg-handler)
                   (values (alist-cons 'manifest arg result)
                           arg-handler)))
         (option '(#\l "list-generations") #f #t
                 (lambda (opt name arg result arg-handler)
                   (values (cons `(query list-generations ,(or arg ""))
                                 result)
                           #f)))
         (option '(#\d "delete-generations") #f #t
                 (lambda (opt name arg result arg-handler)
                   (values (alist-cons 'delete-generations (or arg "")
                                       result)
                           #f)))
         (option '(#\S "switch-generation") #t #f
                 (lambda (opt name arg result arg-handler)
                   (values (alist-cons 'switch-generation arg result)
                           #f)))
         (option '("search-paths") #f #t
                 (lambda (opt name arg result arg-handler)
                   (let ((kind (match arg
                                 ((or "exact" "prefix" "suffix")
                                  (string->symbol arg))
                                 (#f
                                  'exact)
                                 (x
                                  (leave (G_ "~a: unsupported \
kind of search path~%")
                                         x)))))
                     (values (cons `(query search-paths ,kind)
                                   result)
                             #f))))
         (option '(#\p "profile") #t #f
                 (lambda (opt name arg result arg-handler)
                   (values (alist-cons 'profile (canonicalize-profile arg)
                                       result)
                           #f)))
         (option '(#\n "dry-run") #f #f
                 (lambda (opt name arg result arg-handler)
                   (values (alist-cons 'dry-run? #t
                                       (alist-cons 'graft? #f result))
                           #f)))
         (option '("bootstrap") #f #f
                 (lambda (opt name arg result arg-handler)
                   (values (alist-cons 'bootstrap? #t result)
                           #f)))
         (option '("verbose") #f #f
                 (lambda (opt name arg result arg-handler)
                   (values (alist-cons 'verbose? #t result)
                           #f)))
         (option '("allow-collisions") #f #f
                 (lambda (opt name arg result arg-handler)
                   (values (alist-cons 'allow-collisions? #t result)
                           #f)))
         (option '(#\s "search") #t #f
                 (lambda (opt name arg result arg-handler)
                   (values (cons `(query search ,(or arg ""))
                                 result)
                           #f)))
         (option '(#\I "list-installed") #f #t
                 (lambda (opt name arg result arg-handler)
                   (values (cons `(query list-installed ,(or arg ""))
                                 result)
                           #f)))
         (option '(#\A "list-available") #f #t
                 (lambda (opt name arg result arg-handler)
                   (values (cons `(query list-available ,(or arg ""))
                                 result)
                           #f)))
         (option '("show") #t #t
                 (lambda (opt name arg result arg-handler)
                   (values (cons `(query show ,arg)
                                 result)
                           #f)))

         (append %transformation-options
                 %standard-build-options)))

(define (options->upgrade-predicate opts)
  "Return a predicate based on the upgrade/do-not-upgrade regexps in OPTS
that, given a package name, returns true if the package is a candidate for
upgrading, #f otherwise."
  (define upgrade-regexps
    (filter-map (match-lambda
                  (('upgrade . regexp)
                   (make-regexp* (or regexp "")))
                  (_ #f))
                opts))

  (define do-not-upgrade-regexps
    (filter-map (match-lambda
                  (('do-not-upgrade . regexp)
                   (make-regexp* regexp))
                  (_ #f))
                opts))

  (lambda (name)
    (and (any (cut regexp-exec <> name) upgrade-regexps)
         (not (any (cut regexp-exec <> name) do-not-upgrade-regexps)))))

(define (store-item->manifest-entry item)
  "Return a manifest entry for ITEM, a \"/gnu/store/...\" file name."
  (let-values (((name version)
                (package-name->name+version (store-path-package-name item)
                                            #\-)))
    (manifest-entry
      (name name)
      (version version)
      (output "out")                              ;XXX: wild guess
      (item item))))

(define (package-provenance package)
  "Return the provenance of PACKAGE as an sexp for use as the 'provenance'
property of manifest entries, or #f if it could not be determined."
  (define (entry-source entry)
    (match (assq 'source
                 (manifest-entry-properties entry))
      (('source value) value)
      (_ #f)))

  (match (and=> (package-location package) location-file)
    (#f #f)
    (file
     (let ((file (if (string-prefix? "/" file)
                     file
                     (search-path %load-path file))))
       (and file
            (string-prefix? (%store-prefix) file)

            ;; Always store information about the 'guix' channel and
            ;; optionally about the specific channel FILE comes from.
            (or (let ((main  (and=> (find (lambda (entry)
                                            (string=? "guix"
                                                      (manifest-entry-name entry)))
                                          (current-profile-entries))
                                    entry-source))
                      (extra (any (lambda (entry)
                                    (let ((item (manifest-entry-item entry)))
                                      (and (string-prefix? item file)
                                           (entry-source entry))))
                                  (current-profile-entries))))
                  (and main
                       `(,main
                         ,@(if extra (list extra) '()))))))))))

(define (package->manifest-entry* package output)
  "Like 'package->manifest-entry', but attach PACKAGE provenance meta-data to
the resulting manifest entry."
  (define (provenance-properties package)
    (match (package-provenance package)
      (#f   '())
      (sexp `((provenance ,@sexp)))))

  (package->manifest-entry package output
                           #:properties (provenance-properties package)))

(define (options->installable opts manifest transaction)
  "Given MANIFEST, the current manifest, and OPTS, the result of 'args-fold',
return an variant of TRANSACTION that accounts for the specified installations
and upgrades."
  (define upgrade?
    (options->upgrade-predicate opts))

  (define upgraded
    (fold-right (lambda (entry transaction)
                  (if (upgrade? (manifest-entry-name entry))
                      (transaction-upgrade-entry entry transaction)
                      transaction))
                transaction
                (manifest-entries manifest)))

  (define to-install
    (filter-map (match-lambda
                  (('install . (? package? p))
                   ;; When given a package via `-e', install the first of its
                   ;; outputs (XXX).
                   (package->manifest-entry* p "out"))
                  (('install . (? string? spec))
                   (if (store-path? spec)
                       (store-item->manifest-entry spec)
                       (let-values (((package output)
                                     (specification->package+output spec)))
                         (package->manifest-entry* package output))))
                  (_ #f))
                opts))

  (fold manifest-transaction-install-entry
        upgraded
        to-install))

(define (options->removable options manifest transaction)
  "Given options, return a variant of TRANSACTION augmented with the list of
patterns of packages to remove."
  (fold (lambda (opt transaction)
          (match opt
            (('remove . spec)
             (call-with-values
                 (lambda ()
                   (package-specification->name+version+output spec))
               (lambda (name version output)
                 (manifest-transaction-remove-pattern
                  (manifest-pattern
                    (name name)
                    (version version)
                    (output output))
                  transaction))))
            (_ transaction)))
        transaction
        options))

(define (register-gc-root store profile)
  "Register PROFILE, a profile generation symlink, as a GC root, unless it
doesn't need it."
  (define absolute
    ;; We must pass the daemon an absolute file name for PROFILE.  However, we
    ;; cannot use (canonicalize-path profile) because that would return us the
    ;; target of PROFILE in the store; using a store item as an indirect root
    ;; would mean that said store item will always remain live, which is not
    ;; what we want here.
    (if (string-prefix? "/" profile)
        profile
        (string-append (getcwd) "/" profile)))

  (add-indirect-root store absolute))

;;;
;;; Queries and actions.
;;;

(define (process-query opts)
  "Process any query specified by OPTS.  Return #t when a query was actually
processed, #f otherwise."
  (let* ((profiles (match (filter-map (match-lambda
                                        (('profile . p) p)
                                        (_              #f))
                                      opts)
                     (() (list %current-profile))
                     (lst (reverse lst))))
         (profile  (match profiles
                     ((head tail ...) head))))
    (match (assoc-ref opts 'query)
      (('list-generations pattern)
       (define (list-generation display-function number)
         (unless (zero? number)
           (display-generation profile number)
           (display-function profile number)
           (newline)))
       (define (diff-profiles profile numbers)
         (unless (null-list? (cdr numbers))
           (display-profile-content-diff profile (car numbers) (cadr numbers))
           (diff-profiles profile (cdr numbers))))

       (leave-on-EPIPE
        (cond ((not (file-exists? profile))       ; XXX: race condition
               (raise (condition (&profile-not-found-error
                                  (profile profile)))))
              ((string-null? pattern)
               (match (profile-generations profile)
                 (()
                  #t)
                 ((first rest ...)
                  (list-generation display-profile-content first)
                  (diff-profiles profile (cons first rest)))))
              ((matching-generations pattern profile)
               =>
               (lambda (numbers)
                 (if (null-list? numbers)
                     (exit 1)
                     (begin
                       (list-generation display-profile-content (car numbers))
                       (diff-profiles profile numbers)))))
              (else
               (leave (G_ "invalid syntax: ~a~%")
                      pattern))))
       #t)

      (('list-installed regexp)
       (let* ((regexp    (and regexp (make-regexp* regexp)))
              (manifest  (profile-manifest profile))
              (installed (manifest-entries manifest)))
         (leave-on-EPIPE
          (for-each (match-lambda
                      (($ <manifest-entry> name version output path _)
                       (when (or (not regexp)
                                 (regexp-exec regexp name))
                         (format #t "~a\t~a\t~a\t~a~%"
                                 name (or version "?") output path))))

                    ;; Show most recently installed packages last.
                    (reverse installed)))
         #t))

      (('list-available regexp)
       (let* ((regexp    (and regexp (make-regexp* regexp)))
              (available (fold-packages
                          (lambda (p r)
                            (let ((n (package-name p)))
                              (if (and (supported-package? p)
                                       (not (package-superseded p)))
                                  (if regexp
                                      (if (regexp-exec regexp n)
                                          (cons p r)
                                          r)
                                      (cons p r))
                                  r)))
                          '())))
         (leave-on-EPIPE
          (for-each (lambda (p)
                      (format #t "~a\t~a\t~a\t~a~%"
                              (package-name p)
                              (package-version p)
                              (string-join (package-outputs p) ",")
                              (location->string (package-location p))))
                    (sort available
                          (lambda (p1 p2)
                            (string<? (package-name p1)
                                      (package-name p2))))))
         #t))

      (('search _)
       (let* ((patterns (filter-map (match-lambda
                                      (('query 'search rx) rx)
                                      (_                   #f))
                                    opts))
              (regexps  (map (cut make-regexp* <> regexp/icase) patterns)))
         (leave-on-EPIPE
          (let-values (((packages scores)
                        (find-packages-by-description regexps)))
            (for-each (lambda (package score)
                        (package->recutils package (current-output-port)
                                           #:extra-fields
                                           `((relevance . ,score))))
                      packages
                      scores)))
         #t))

      (('show requested-name)
       (let-values (((name version)
                     (package-name->name+version requested-name)))
         (leave-on-EPIPE
          (for-each (cute package->recutils <> (current-output-port))
                    (find-packages-by-name name version)))
         #t))

      (('search-paths kind)
       (let* ((manifests (map profile-manifest profiles))
              (entries   (append-map manifest-transitive-entries
                                     manifests))
              (profiles  (map user-friendly-profile profiles))
              (settings  (search-path-environment-variables entries profiles
                                                            (const #f)
                                                            #:kind kind)))
         (format #t "~{~a~%~}" settings)
         #t))

      (_ #f))))

(define* (roll-back-action store profile arg opts
                           #:key dry-run?)
  "Roll back PROFILE to its previous generation."
  (unless dry-run?
    (roll-back* store profile)))

(define* (switch-generation-action store profile spec opts
                                   #:key dry-run?)
  "Switch PROFILE to the generation specified by SPEC."
  (unless dry-run?
    (let ((number (relative-generation-spec->number profile spec)))
      (if number
          (switch-to-generation* profile number)
          (leave (G_ "cannot switch to generation '~a'~%") spec)))))

(define* (delete-generations-action store profile pattern opts
                                    #:key dry-run?)
  "Delete PROFILE's generations that match PATTERN."
  (unless dry-run?
    (delete-matching-generations store profile pattern)))

(define* (manifest-action store profile file opts
                          #:key dry-run?)
  "Change PROFILE to contain the packages specified in FILE."
  (let* ((user-module  (make-user-module '((guix profiles) (gnu))))
         (manifest     (load* file user-module))
         (bootstrap?   (assoc-ref opts 'bootstrap?))
         (substitutes? (assoc-ref opts 'substitutes?))
         (allow-collisions? (assoc-ref opts 'allow-collisions?)))
    (if dry-run?
        (format #t (G_ "would install new manifest from '~a' with ~d entries~%")
                file (length (manifest-entries manifest)))
        (format #t (G_ "installing new manifest from '~a' with ~d entries~%")
                file (length (manifest-entries manifest))))
    (build-and-use-profile store profile manifest
                           #:allow-collisions? allow-collisions?
                           #:bootstrap? bootstrap?
                           #:use-substitutes? substitutes?
                           #:dry-run? dry-run?)))

(define %actions
  ;; List of actions that may be processed.  The car of each pair is the
  ;; action's symbol in the option list; the cdr is the action's procedure.
  `((roll-back? . ,roll-back-action)
    (switch-generation . ,switch-generation-action)
    (delete-generations . ,delete-generations-action)
    (manifest . ,manifest-action)))

(define (process-actions store opts)
  "Process any install/remove/upgrade action from OPTS."

  (define dry-run? (assoc-ref opts 'dry-run?))
  (define bootstrap? (assoc-ref opts 'bootstrap?))
  (define substitutes? (assoc-ref opts 'substitutes?))
  (define allow-collisions? (assoc-ref opts 'allow-collisions?))
  (define profile  (or (assoc-ref opts 'profile) %current-profile))
  (define transform (options->transformation opts))

  (define (transform-entry entry)
    (let ((item (transform store (manifest-entry-item entry))))
      (manifest-entry
        (inherit entry)
        (item item)
        (version (if (package? item)
                     (package-version item)
                     (manifest-entry-version entry))))))

  ;; First, process roll-backs, generation removals, etc.
  (for-each (match-lambda
              ((key . arg)
               (and=> (assoc-ref %actions key)
                      (lambda (proc)
                        (proc store profile arg opts
                              #:dry-run? dry-run?)))))
            opts)

  ;; Then, process normal package removal/installation/upgrade.
  (let* ((manifest (profile-manifest profile))
         (step1    (options->removable opts manifest
                                       (manifest-transaction)))
         (step2    (options->installable opts manifest step1))
         (step3    (manifest-transaction
                    (inherit step2)
                    (install (map transform-entry
                                  (manifest-transaction-install step2)))))
         (new      (manifest-perform-transaction manifest step3)))

    (warn-about-old-distro)

    (unless (manifest-transaction-null? step3)
      (show-manifest-transaction store manifest step3
                                 #:dry-run? dry-run?)
      (build-and-use-profile store profile new
                             #:allow-collisions? allow-collisions?
                             #:bootstrap? bootstrap?
                             #:use-substitutes? substitutes?
                             #:dry-run? dry-run?))))

;;;
;;; Entry point.
;;;

(define (guix-package . args)
  (define (handle-argument arg result arg-handler)
    ;; Process non-option argument ARG by calling back ARG-HANDLER.
    (if arg-handler
        (arg-handler arg result)
        (leave (G_ "~A: extraneous argument~%") arg)))

  (define opts
    (parse-command-line args %options (list %default-options #f)
                        #:argument-handler handle-argument))
  (define verbose?
    (assoc-ref opts 'verbose?))

  (with-error-handling
    (or (process-query opts)
        (parameterize ((%store  (open-connection))
                       (%graft? (assoc-ref opts 'graft?)))
          (with-status-report print-build-event/quiet
            (set-build-options-from-command-line (%store) opts)
            (parameterize ((%guile-for-build
                            (package-derivation
                             (%store)
                             (if (assoc-ref opts 'bootstrap?)
                                 %bootstrap-guile
                                 (canonical-package guile-2.2)))))
              (process-actions (%store) opts)))))))


code:
;;; GNU Guix --- Functional package management for GNU
;;; Copyright © 2016, 2017, 2018 Ludovic Courtès <ludo@gnu.org>
;;;
;;; This file is part of GNU Guix.
;;;
;;; GNU Guix is free software; you can redistribute it and/or modify it
;;; under the terms of the GNU General Public License as published by
;;; the Free Software Foundation; either version 3 of the License, or (at
;;; your option) any later version.
;;;
;;; GNU Guix is distributed in the hope that it will be useful, but
;;; WITHOUT ANY WARRANTY; without even the implied warranty of
;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
;;; GNU General Public License for more details.
;;;
;;; You should have received a copy of the GNU General Public License
;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.

(define-module (guix scripts perform-download)
  #:use-module (guix ui)
  #:use-module (guix derivations)
  #:use-module ((guix store) #:select (derivation-path? store-path?))
  #:use-module (guix build download)
  #:use-module (ice-9 match)
  #:export (guix-perform-download))

;; This program is a helper for the daemon's 'download' built-in builder.

(define-syntax derivation-let
  (syntax-rules ()
    ((_ drv ((id name) rest ...) body ...)
     (let ((id (assoc-ref (derivation-builder-environment-vars drv)
                          name)))
       (derivation-let drv (rest ...) body ...)))
    ((_ drv () body ...)
     (begin body ...))))

(define %user-module
  ;; Module in which content-address mirror procedures are evaluated.
  (let ((module (make-fresh-user-module)))
    (module-use! module (resolve-interface '(guix base32)))
    module))

(define* (perform-download drv #:optional output
                           #:key print-build-trace?)
  "Perform the download described by DRV, a fixed-output derivation, to
OUTPUT.

Note: Unless OUTPUT is #f, we don't read the value of 'out' in DRV since the
actual output is different from that when we're doing a 'bmCheck' or
'bmRepair' build."
  (derivation-let drv ((url "url")
                       (output* "out")
                       (executable "executable")
                       (mirrors "mirrors")
                       (content-addressed-mirrors "content-addressed-mirrors"))
    (unless url
      (leave (G_ "~a: missing URL~%") (derivation-file-name drv)))

    (let* ((output     (or output output*))
           (url        (call-with-input-string url read))
           (drv-output (assoc-ref (derivation-outputs drv) "out"))
           (algo       (derivation-output-hash-algo drv-output))
           (hash       (derivation-output-hash drv-output)))
      (unless (and algo hash)
        (leave (G_ "~a is not a fixed-output derivation~%")
               (derivation-file-name drv)))

      ;; We're invoked by the daemon, which gives us write access to OUTPUT.
      (when (url-fetch url output
                       #:print-build-trace? print-build-trace?
                       #:mirrors (if mirrors
                                     (call-with-input-file mirrors read)
                                     '())
                       #:content-addressed-mirrors
                       (if content-addressed-mirrors
                           (call-with-input-file content-addressed-mirrors
                             (lambda (port)
                               (eval (read port) %user-module)))
                           '())
                       #:hashes `((,algo . ,hash))

                       ;; Since DRV's output hash is known, X.509 certificate
                       ;; validation is pointless.
                       #:verify-certificate? #f)
        (when (and executable (string=? executable "1"))
          (chmod output #o755))))))

(define (assert-low-privileges)
  (when (zero? (getuid))
    (leave (G_ "refusing to run with elevated privileges (UID ~a)~%")
           (getuid))))

(define (guix-perform-download . args)
  "Perform the download described by the given fixed-output derivation.

This is an \"out-of-band\" download in that this code is executed directly by
the daemon and not explicitly described as an input of the derivation.  This
allows us to sidestep bootstrapping problems, such downloading the source code
of GnuTLS over HTTPS, before we have built GnuTLS.  See
<http://bugs.gnu.org/22774>."

  (define print-build-trace?
    (match (getenv "_NIX_OPTIONS")
      (#f #f)
      (str (string-contains str "print-extended-build-trace=1"))))

  ;; This program must be invoked by guix-daemon under an unprivileged UID to
  ;; prevent things downloading from 'file:///etc/shadow' or arbitrary code
  ;; execution via the content-addressed mirror procedures.  (That means we
  ;; exclude users who did not pass '--build-users-group'.)
  (with-error-handling
    (match args
      (((? derivation-path? drv) (? store-path? output))
       (assert-low-privileges)
       (perform-download (read-derivation-from-file drv)
                         output
                         #:print-build-trace? print-build-trace?))
      (((? derivation-path? drv))                 ;backward compatibility
       (assert-low-privileges)
       (perform-download (read-derivation-from-file drv)
                         #:print-build-trace? print-build-trace?))
      (("--version")
       (show-version-and-exit))
      (x
       (leave
        (G_ "fixed-output derivation and output file name expected~%"))))))

;; Local Variables:
;; eval: (put 'derivation-let 'scheme-indent-function 2)
;; End:

;; perform-download.scm ends here

license: mit


code:
;;; GNU Guix --- Functional package management for GNU
;;; Copyright © 2015 David Thompson <davet@gnu.org>
;;; Copyright © 2015, 2016, 2017, 2018 Ludovic Courtès <ludo@gnu.org>
;;;
;;; This file is part of GNU Guix.
;;;
;;; GNU Guix is free software; you can redistribute it and/or modify it
;;; under the terms of the GNU General Public License as published by
;;; the Free Software Foundation; either version 3 of the License, or (at
;;; your option) any later version.
;;;
;;; GNU Guix is distributed in the hope that it will be useful, but
;;; WITHOUT ANY WARRANTY; without even the implied warranty of
;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
;;; GNU General Public License for more details.
;;;
;;; You should have received a copy of the GNU General Public License
;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.

(define-module (guix scripts publish)
  #:use-module ((system repl server) #:prefix repl:)
  #:use-module (ice-9 binary-ports)
  #:use-module (ice-9 format)
  #:use-module (ice-9 match)
  #:use-module (ice-9 regex)
  #:use-module (ice-9 rdelim)
  #:use-module (ice-9 threads)
  #:use-module (rnrs bytevectors)
  #:use-module (srfi srfi-1)
  #:use-module (srfi srfi-2)
  #:use-module (srfi srfi-9)
  #:use-module (srfi srfi-9 gnu)
  #:use-module (srfi srfi-19)
  #:use-module (srfi srfi-26)
  #:use-module (srfi srfi-34)
  #:use-module (srfi srfi-37)
  #:use-module (web http)
  #:use-module (web request)
  #:use-module (web response)
  #:use-module (web server)
  #:use-module (web uri)
  #:autoload   (sxml simple) (sxml->xml)
  #:use-module (guix base32)
  #:use-module (guix base64)
  #:use-module (guix config)
  #:use-module (guix derivations)
  #:use-module (gcrypt hash)
  #:use-module (guix pki)
  #:use-module (gcrypt pk-crypto)
  #:use-module (guix workers)
  #:use-module (guix store)
  #:use-module ((guix serialization) #:select (write-file))
  #:use-module (guix zlib)
  #:use-module (guix cache)
  #:use-module (guix ui)
  #:use-module (guix scripts)
  #:use-module ((guix utils)
                #:select (with-atomic-file-output compressed-file?))
  #:use-module ((guix build utils)
                #:select (dump-port mkdir-p find-files))
  #:use-module ((guix build syscalls) #:select (set-thread-name))
  #:export (%public-key
            %private-key

            guix-publish))

(define (show-help)
  (format #t (G_ "Usage: guix publish [OPTION]...
Publish ~a over HTTP.\n") %store-directory)
  (display (G_ "
  -p, --port=PORT        listen on PORT"))
  (display (G_ "
      --listen=HOST      listen on the network interface for HOST"))
  (display (G_ "
  -u, --user=USER        change privileges to USER as soon as possible"))
  (display (G_ "
  -C, --compression[=LEVEL]
                         compress archives at LEVEL"))
  (display (G_ "
  -c, --cache=DIRECTORY  cache published items to DIRECTORY"))
  (display (G_ "
      --workers=N        use N workers to bake items"))
  (display (G_ "
      --ttl=TTL          announce narinfos can be cached for TTL seconds"))
  (display (G_ "
      --nar-path=PATH    use PATH as the prefix for nar URLs"))
  (display (G_ "
      --public-key=FILE  use FILE as the public key for signatures"))
  (display (G_ "
      --private-key=FILE use FILE as the private key for signatures"))
  (display (G_ "
  -r, --repl[=PORT]      spawn REPL server on PORT"))
  (newline)
  (display (G_ "
  -h, --help             display this help and exit"))
  (display (G_ "
  -V, --version          display version information and exit"))
  (newline)
  (show-bug-report-information))

(define (getaddrinfo* host)
  "Like 'getaddrinfo', but properly report errors."
  (catch 'getaddrinfo-error
    (lambda ()
      (getaddrinfo host))
    (lambda (key error)
      (leave (G_ "lookup of host '~a' failed: ~a~%")
             host (gai-strerror error)))))

;; Nar compression parameters.
(define-record-type <compression>
  (compression type level)
  compression?
  (type   compression-type)
  (level  compression-level))

(define %no-compression
  (compression 'none 0))

(define %default-gzip-compression
  ;; Since we compress on the fly, default to fast compression.
  (compression 'gzip 3))

(define (actual-compression item requested)
  "Return the actual compression used for ITEM, which may be %NO-COMPRESSION
if ITEM is already compressed."
  (if (compressed-file? item)
      %no-compression
      requested))

(define %options
  (list (option '(#\h "help") #f #f
                (lambda _
                  (show-help)
                  (exit 0)))
        (option '(#\V "version") #f #f
                (lambda _
                  (show-version-and-exit "guix publish")))
        (option '(#\u "user") #t #f
                (lambda (opt name arg result)
                  (alist-cons 'user arg result)))
        (option '(#\p "port") #t #f
                (lambda (opt name arg result)
                  (alist-cons 'port (string->number* arg) result)))
        (option '("listen") #t #f
                (lambda (opt name arg result)
                  (match (getaddrinfo* arg)
                    ((info _ ...)
                     (alist-cons 'address (addrinfo:addr info)
                                 result))
                    (()
                     (leave (G_ "lookup of host '~a' returned nothing")
                            name)))))
        (option '(#\C "compression") #f #t
                (lambda (opt name arg result)
                  (match (if arg (string->number* arg) 3)
                    (0
                     (alist-cons 'compression %no-compression result))
                    (level
                     (if (zlib-available?)
                         (alist-cons 'compression
                                     (compression 'gzip level)
                                     result)
                         (begin
                           (warning (G_ "zlib support is missing; \
compression disabled~%"))
                           result))))))
        (option '(#\c "cache") #t #f
                (lambda (opt name arg result)
                  (alist-cons 'cache arg result)))
        (option '("workers") #t #f
                (lambda (opt name arg result)
                  (alist-cons 'workers (string->number* arg)
                              result)))
        (option '("ttl") #t #f
                (lambda (opt name arg result)
                  (let ((duration (string->duration arg)))
                    (unless duration
                      (leave (G_ "~a: invalid duration~%") arg))
                    (alist-cons 'narinfo-ttl (time-second duration)
                                result))))
        (option '("nar-path") #t #f
                (lambda (opt name arg result)
                  (alist-cons 'nar-path arg result)))
        (option '("public-key") #t #f
                (lambda (opt name arg result)
                  (alist-cons 'public-key-file arg result)))
        (option '("private-key" "secret-key") #t #f
                (lambda (opt name arg result)
                  (alist-cons 'private-key-file arg result)))
        (option '(#\r "repl") #f #t
                (lambda (opt name arg result)
                  ;; If port unspecified, use default Guile REPL port.
                  (let ((port (and arg (string->number* arg))))
                    (alist-cons 'repl (or port 37146) result))))))

(define %default-options
  `((port . 8080)

    ;; By default, serve nars under "/nar".
    (nar-path . "nar")

    (public-key-file . ,%public-key-file)
    (private-key-file . ,%private-key-file)

    ;; Default to fast & low compression.
    (compression . ,(if (zlib-available?)
                        %default-gzip-compression
                        %no-compression))

    ;; Default number of workers when caching is enabled.
    (workers . ,(current-processor-count))

    (address . ,(make-socket-address AF_INET INADDR_ANY 0))
    (repl . #f)))

;; The key pair used to sign narinfos.
(define %private-key
  (make-parameter #f))
(define %public-key
  (make-parameter #f))

(define %nix-cache-info
  `(("StoreDir" . ,%store-directory)
    ("WantMassQuery" . 0)
    ("Priority" . 100)))

(define (signed-string s)
  "Sign the hash of the string S with the daemon's key."
  (let* ((public-key (%public-key))
         (hash (bytevector->hash-data (sha256 (string->utf8 s))
                                      #:key-type (key-type public-key))))
    (signature-sexp hash (%private-key) public-key)))

(define base64-encode-string
  (compose base64-encode string->utf8))

(define* (narinfo-string store store-path key
                         #:key (compression %no-compression)
                         (nar-path "nar") file-size)
  "Generate a narinfo key/value string for STORE-PATH; an exception is raised
if STORE-PATH is invalid.  Produce a URL that corresponds to COMPRESSION.  The
narinfo is signed with KEY.  NAR-PATH specifies the prefix for nar URLs.
Optionally, FILE-SIZE can specify the size in bytes of the compressed NAR; it
informs the client of how much needs to be downloaded."
  (let* ((path-info  (query-path-info store store-path))
         (compression (actual-compression store-path compression))
         (url        (encode-and-join-uri-path
                      `(,@(split-and-decode-uri-path nar-path)
                        ,@(match compression
                            (($ <compression> 'none)
                             '())
                            (($ <compression> type)
                             (list (symbol->string type))))
                        ,(basename store-path))))
         (hash       (bytevector->nix-base32-string
                      (path-info-hash path-info)))
         (size       (path-info-nar-size path-info))
         (file-size  (or file-size
                         (and (eq? compression %no-compression) size)))
         (references (string-join
                      (map basename (path-info-references path-info))
                      " "))
         (deriver    (path-info-deriver path-info))
         (base-info  (format #f
                             "\
StorePath: ~a
URL: ~a
Compression: ~a
NarHash: sha256:~a
NarSize: ~d
References: ~a~%~a"
                             store-path url
                             (compression-type compression)
                             hash size references
                             (if file-size
                                 (format #f "FileSize: ~a~%" file-size)
                                 "")))
         ;; Do not render a "Deriver" or "System" line if we are rendering
         ;; info for a derivation.
         (info       (if (not deriver)
                         base-info
                         (catch 'system-error
                           (lambda ()
                             (let ((drv (read-derivation-from-file deriver)))
                               (format #f "~aSystem: ~a~%Deriver: ~a~%"
                                       base-info (derivation-system drv)
                                       (basename deriver))))
                           (lambda args
                             ;; DERIVER might be missing, but that's fine:
                             ;; it's only used for <substitutable> where it's
                             ;; optional.  'System' is currently unused.
                             (if (= ENOENT (system-error-errno args))
                                 base-info
                                 (apply throw args))))))
         (signature  (base64-encode-string
                      (canonical-sexp->string (signed-string info)))))
    (format #f "~aSignature: 1;~a;~a~%" info (gethostname) signature)))

(define* (not-found request
                    #:key (phrase "Resource not found")
                    ttl)
  "Render 404 response for REQUEST."
  (values (build-response #:code 404
                          #:headers (if ttl
                                        `((cache-control (max-age . ,ttl)))
                                        '()))
          (string-append phrase ": "
                         (uri-path (request-uri request)))))

(define (render-nix-cache-info)
  "Render server information."
  (values '((content-type . (text/plain)))
          (lambda (port)
            (for-each (match-lambda
                       ((key . value)
                        (format port "~a: ~a~%" key value)))
                      %nix-cache-info))))

(define* (render-narinfo store request hash
                         #:key ttl (compression %no-compression)
                         (nar-path "nar"))
  "Render metadata for the store path corresponding to HASH.  If TTL is true,
advertise it as the maximum validity period (in seconds) via the
'Cache-Control' header.  This allows 'guix substitute' to cache it for an
appropriate duration.  NAR-PATH specifies the prefix for nar URLs."
  (let ((store-path (hash-part->path store hash)))
    (if (string-null? store-path)
        (not-found request #:phrase "")
        (values `((content-type . (application/x-nix-narinfo))
                  ,@(if ttl
                        `((cache-control (max-age . ,ttl)))
                        '()))
                (cut display
                  (narinfo-string store store-path (%private-key)
                                  #:nar-path nar-path
                                  #:compression compression)
                  <>)))))

(define* (nar-cache-file directory item
                             #:key (compression %no-compression))
  (string-append directory "/"
                 (symbol->string (compression-type compression))
                 "/" (basename item) ".nar"))

(define* (narinfo-cache-file directory item
                             #:key (compression %no-compression))
  (string-append directory "/"
                 (symbol->string (compression-type compression))
                 "/" (basename item)
                 ".narinfo"))

(define run-single-baker
  (let ((baking (make-weak-value-hash-table))
        (mutex  (make-mutex)))
    (lambda (item thunk)
      "Run THUNK, which is supposed to bake ITEM, but make sure only one
thread is baking ITEM at a given time."
      (define selected?
        (with-mutex mutex
          (and (not (hash-ref baking item))
               (begin
                 (hash-set! baking item (current-thread))
                 #t))))

      (when selected?
        (dynamic-wind
          (const #t)
          thunk
          (lambda ()
            (with-mutex mutex
              (hash-remove! baking item))))))))

(define-syntax-rule (single-baker item exp ...)
  "Bake ITEM by evaluating EXP, but make sure there's only one baker for ITEM
at a time."
  (run-single-baker item (lambda () exp ...)))

(define (narinfo-files cache)
  "Return the list of .narinfo files under CACHE."
  (if (file-is-directory? cache)
      (find-files cache
                  (lambda (file stat)
                    (string-suffix? ".narinfo" file)))
      '()))

(define (nar-expiration-time ttl)
  "Return the narinfo expiration time (in seconds since the Epoch).  The
expiration time is +inf.0 when passed an item that is still in the store; in
other cases, it is the last-access time of the item plus TTL.

This policy allows us to keep cached nars that correspond to valid store
items.  Failing that, we could eventually have to recompute them and return
404 in the meantime."
  (let ((expiration-time (file-expiration-time ttl)))
    (lambda (file)
      (let ((item (string-append (%store-prefix) "/"
                                 (basename file ".narinfo"))))
        ;; Note: We don't need to use 'valid-path?' here because FILE would
        ;; not exist if ITEM were not valid in the first place.
        (if (file-exists? item)
            +inf.0
            (expiration-time file))))))

(define* (render-narinfo/cached store request hash
                                #:key ttl (compression %no-compression)
                                (nar-path "nar")
                                cache pool)
  "Respond to the narinfo request for REQUEST.  If the narinfo is available in
CACHE, then send it; otherwise, return 404 and \"bake\" that nar and narinfo
requested using POOL."
  (define (delete-entry narinfo)
    ;; Delete NARINFO and the corresponding nar from CACHE.
    (let ((nar (string-append (string-drop-right narinfo
                                                 (string-length ".narinfo"))
                              ".nar")))
      (delete-file* narinfo)
      (delete-file* nar)))

  (let* ((item        (hash-part->path store hash))
         (compression (actual-compression item compression))
         (cached      (and (not (string-null? item))
                           (narinfo-cache-file cache item
                                               #:compression compression))))
    (cond ((string-null? item)
           (not-found request))
          ((file-exists? cached)
           ;; Narinfo is in cache, send it.
           (values `((content-type . (application/x-nix-narinfo))
                     ,@(if ttl
                           `((cache-control (max-age . ,ttl)))
                           '()))
                   (lambda (port)
                     (display (call-with-input-file cached
                                read-string)
                              port))))
          ((and (file-exists? item)        ;cheaper than the 'valid-path?' RPC
                (valid-path? store item))
           ;; Nothing in cache: bake the narinfo and nar in the background and
           ;; return 404.
           (eventually pool
             (single-baker item
               ;; Check whether CACHED has been produced in the meantime.
               (unless (file-exists? cached)
                 ;; (format #t "baking ~s~%" item)
                 (bake-narinfo+nar cache item
                                   #:ttl ttl
                                   #:compression compression
                                   #:nar-path nar-path)))

             (when ttl
               (single-baker 'cache-cleanup
                 (maybe-remove-expired-cache-entries cache
                                                     narinfo-files
                                                     #:entry-expiration
                                                     (nar-expiration-time ttl)
                                                     #:delete-entry delete-entry
                                                     #:cleanup-period ttl))))
           (not-found request
                      #:phrase "We're baking it"
                      #:ttl 300))              ;should be available within 5m
          (else
           (not-found request #:phrase "")))))

(define* (bake-narinfo+nar cache item
                           #:key ttl (compression %no-compression)
                           (nar-path "/nar"))
  "Write the narinfo and nar for ITEM to CACHE."
  (let* ((compression (actual-compression item compression))
         (nar         (nar-cache-file cache item
                                      #:compression compression))
         (narinfo     (narinfo-cache-file cache item
                                          #:compression compression)))

    (mkdir-p (dirname nar))
    (match (compression-type compression)
      ('gzip
       ;; Note: the file port gets closed along with the gzip port.
       (call-with-gzip-output-port (open-output-file (string-append nar ".tmp"))
         (lambda (port)
           (write-file item port))
         #:level (compression-level compression)
         #:buffer-size (* 128 1024))
       (rename-file (string-append nar ".tmp") nar))
      ('none
       ;; Cache nars even when compression is disabled so that we can
       ;; guarantee the TTL (see <https://bugs.gnu.org/28664>.)
       (with-atomic-file-output nar
         (lambda (port)
           (write-file item port)))))

    (mkdir-p (dirname narinfo))
    (with-atomic-file-output narinfo
      (lambda (port)
        ;; Open a new connection to the store.  We cannot reuse the main
        ;; thread's connection to the store since we would end up sending
        ;; stuff concurrently on the same channel.
        (with-store store
          (display (narinfo-string store item
                                   (%private-key)
                                   #:nar-path nar-path
                                   #:compression compression
                                   #:file-size (and=> (stat nar #f)
                                                      stat:size))
                   port))))))

;; XXX: Declare the 'X-Nar-Compression' HTTP header, which is in fact for
;; internal consumption: it allows us to pass the compression info to
;; 'http-write', as part of the workaround to <http://bugs.gnu.org/21093>.
(declare-header! "X-Nar-Compression"
                 (lambda (str)
                   (match (call-with-input-string str read)
                     (('compression type level)
                      (compression type level))))
                 compression?
                 (lambda (compression port)
                   (match compression
                     (($ <compression> type level)
                      (write `(compression ,type ,level) port)))))

(define* (render-nar store request store-item
                     #:key (compression %no-compression))
  "Render archive of the store path corresponding to STORE-ITEM."
  (let ((store-path (string-append %store-directory "/" store-item)))
    ;; The ISO-8859-1 charset *must* be used otherwise HTTP clients will
    ;; interpret the byte stream as UTF-8 and arbitrarily change invalid byte
    ;; sequences.
    (if (valid-path? store store-path)
        (values `((content-type . (application/x-nix-archive
                                   (charset . "ISO-8859-1")))
                  (x-nar-compression . ,compression))
                ;; XXX: We're not returning the actual contents, deferring
                ;; instead to 'http-write'.  This is a hack to work around
                ;; <http://bugs.gnu.org/21093>.
                store-path)
        (not-found request))))

(define* (render-nar/cached store cache request store-item
                            #:key (compression %no-compression))
  "Respond to REQUEST with a nar for STORE-ITEM.  If the nar is in CACHE,
return it; otherwise, return 404."
  (let ((cached (nar-cache-file cache store-item
                                #:compression compression)))
    (if (file-exists? cached)
        (values `((content-type . (application/octet-stream
                                   (charset . "ISO-8859-1")))
                  ;; XXX: We're not returning the actual contents, deferring
                  ;; instead to 'http-write'.  This is a hack to work around
                  ;; <http://bugs.gnu.org/21093>.
                  (x-raw-file . ,cached))
                #f)
        (not-found request))))

(define (render-content-addressed-file store request
                                       name algo hash)
  "Return the content of the result of the fixed-output derivation NAME that
has the given HASH of type ALGO."
  ;; TODO: Support other hash algorithms.
  (if (and (eq? algo 'sha256) (= 32 (bytevector-length hash)))
      (let ((item (fixed-output-path name hash
                                     #:hash-algo algo
                                     #:recursive? #f)))
        (if (valid-path? store item)
            (values `((content-type . (application/octet-stream
                                       (charset . "ISO-8859-1")))
                      ;; XXX: We're not returning the actual contents,
                      ;; deferring instead to 'http-write'.  This is a hack to
                      ;; work around <http://bugs.gnu.org/21093>.
                      (x-raw-file . ,item))
                    #f)
            (not-found request)))
      (not-found request)))

(define (render-log-file store request name)
  "Render the log file for NAME, the base name of a store item.  Don't attempt
to compress or decompress the log file; just return it as-is."
  (define (response-headers file)
    ;; XXX: We're not returning the actual contents, deferring instead to
    ;; 'http-write'.  This is a hack to work around
    ;; <http://bugs.gnu.org/21093>.
    (cond ((string-suffix? ".gz" file)
           `((content-type . (text/plain (charset . "UTF-8")))
             (content-encoding . (gzip))
             (x-raw-file . ,file)))
          ((string-suffix? ".bz2" file)
           `((content-type . (application/x-bzip2
                              (charset . "ISO-8859-1")))
             (x-raw-file . ,file)))
          (else                                   ;uncompressed
           `((content-type . (text/plain (charset . "UTF-8")))
             (x-raw-file . ,file)))))

  (let ((log (log-file store
                       (string-append (%store-prefix) "/" name))))
    (if log
        (values (response-headers log) log)
        (not-found request))))

(define (render-home-page request)
  "Render the home page."
  (values `((content-type . (text/html (charset . "UTF-8"))))
          (call-with-output-string
            (lambda (port)
              (sxml->xml '(html
                           (head (title "GNU Guix Substitute Server"))
                           (body
                            (h1 "GNU Guix Substitute Server")
                            (p "Hi, "
                               (a (@ (href
                                      "https://gnu.org/s/guix/manual/html_node/Invoking-guix-publish.html"))
                                  (tt "guix publish"))
                               " speaking.  Welcome!")))
                         port)))))

(define (extract-narinfo-hash str)
  "Return the hash within the narinfo resource string STR, or false if STR
is invalid."
  (and (string-suffix? ".narinfo" str)
       (let ((base (string-drop-right str 8)))
         (and (string-every %nix-base32-charset base)
              base))))

(define (get-request? request)
  "Return #t if REQUEST uses the GET method."
  (eq? (request-method request) 'GET))

(define (request-path-components request)
  "Split the URI path of REQUEST into a list of component strings.  For
example: \"/foo/bar\" yields '(\"foo\" \"bar\")."
  (split-and-decode-uri-path (uri-path (request-uri request))))

;;;
;;; Server.
;;;

(define %http-write
  (@@ (web server http) http-write))

(define (strip-headers response)
  "Return RESPONSE's headers minus 'Content-Length' and our internal headers."
  (fold alist-delete
        (response-headers response)
        '(content-length x-raw-file x-nar-compression)))

(define (sans-content-length response)
  "Return RESPONSE without its 'content-length' header."
  (set-field response (response-headers)
             (strip-headers response)))

(define (with-content-length response length)
  "Return RESPONSE with a 'content-length' header set to LENGTH."
  (set-field response (response-headers)
             (alist-cons 'content-length length
                         (strip-headers response))))

(define-syntax-rule (swallow-EPIPE exp ...)
  "Swallow EPIPE errors raised by EXP..."
  (catch 'system-error
    (lambda ()
      exp ...)
    (lambda args
      (if (= EPIPE (system-error-errno args))
          (values)
          (apply throw args)))))

(define-syntax-rule (swallow-zlib-error exp ...)
  "Swallow 'zlib-error' exceptions raised by EXP..."
  (catch 'zlib-error
    (lambda ()
      exp ...)
    (const #f)))

(define (nar-response-port response compression)
  "Return a port on which to write the body of RESPONSE, the response of a
/nar request, according to COMPRESSION."
  (match compression
    (($ <compression> 'gzip level)
     ;; Note: We cannot used chunked encoding here because
     ;; 'make-gzip-output-port' wants a file port.
     (make-gzip-output-port (response-port response)
                            #:level level
                            #:buffer-size (* 64 1024)))
    (($ <compression> 'none)
     (response-port response))
    (#f
     (response-port response))))

(define (http-write server client response body)
  "Write RESPONSE and BODY to CLIENT, possibly in a separate thread to avoid
blocking."
  (match (response-content-type response)
    (('application/x-nix-archive . _)
     ;; Sending the the whole archive can take time so do it in a separate
     ;; thread so that the main thread can keep working in the meantime.
     (call-with-new-thread
      (lambda ()
        (set-thread-name "publish nar")
        (let* ((compression (assoc-ref (response-headers response)
                                       'x-nar-compression))
               (response    (write-response (sans-content-length response)
                                            client))
               (port        (begin
                              (force-output client)
                              (nar-response-port response compression))))
          ;; XXX: Given our ugly workaround for <http://bugs.gnu.org/21093> in
          ;; 'render-nar', BODY here is just the file name of the store item.
          ;; We call 'write-file' from here because we know that's the only
          ;; way to avoid building the whole nar in memory, which could
          ;; quickly become a real problem.  As a bonus, we even do
          ;; sendfile(2) directly from the store files to the socket.
          (swallow-zlib-error
           (swallow-EPIPE
            (write-file (utf8->string body) port)))
          (swallow-zlib-error
           (close-port port))
          (values)))))
    (_
     (match (assoc-ref (response-headers response) 'x-raw-file)
       ((? string? file)
        ;; Send a raw file in a separate thread.
        (call-with-new-thread
         (lambda ()
           (set-thread-name "publish file")
           (catch 'system-error
             (lambda ()
               (call-with-input-file file
                 (lambda (input)
                   (let* ((size     (stat:size (stat input)))
                          (response (write-response (with-content-length response
                                                                         size)
                                                    client))
                          (output   (response-port response)))
                     (if (file-port? output)
                         (sendfile output input size)
                         (dump-port input output))
                     (close-port output)
                     (values)))))
             (lambda args
               ;; If the file was GC'd behind our back, that's fine.  Likewise if
               ;; the client closes the connection.
               (unless (memv (system-error-errno args)
                             (list ENOENT EPIPE ECONNRESET))
                 (apply throw args))
               (values))))))
       (#f
        ;; Handle other responses sequentially.
        (%http-write server client response body))))))

(define-server-impl concurrent-http-server
  ;; A variant of Guile's built-in HTTP server that offloads possibly long
  ;; responses to a different thread.
  (@@ (web server http) http-open)
  (@@ (web server http) http-read)
  http-write
  (@@ (web server http) http-close))

(define* (make-request-handler store
                               #:key
                               cache pool
                               narinfo-ttl
                               (nar-path "nar")
                               (compression %no-compression))
  (define nar-path?
    (let ((expected (split-and-decode-uri-path nar-path)))
      (cut equal? expected <>)))

  (lambda (request body)
    (format #t "~a ~a~%"
            (request-method request)
            (uri-path (request-uri request)))
    (if (get-request? request)                    ;reject POST, PUT, etc.
        (match (request-path-components request)
          ;; /nix-cache-info
          (("nix-cache-info")
           (render-nix-cache-info))
          ;; /
          ((or () ("index.html"))
           (render-home-page request))
          ;; /<hash>.narinfo
          (((= extract-narinfo-hash (? string? hash)))
           ;; TODO: Register roots for HASH that will somehow remain for
           ;; NARINFO-TTL.
           (if cache
               (render-narinfo/cached store request hash
                                      #:cache cache
                                      #:pool pool
                                      #:ttl narinfo-ttl
                                      #:nar-path nar-path
                                      #:compression compression)
               (render-narinfo store request hash
                               #:ttl narinfo-ttl
                               #:nar-path nar-path
                               #:compression compression)))
          ;; /nar/file/NAME/sha256/HASH
          (("file" name "sha256" hash)
           (guard (c ((invalid-base32-character? c)
                      (not-found request)))
             (let ((hash (nix-base32-string->bytevector hash)))
               (render-content-addressed-file store request
                                              name 'sha256 hash))))

          ;; /log/OUTPUT
          (("log" name)
           (render-log-file store request name))

          ;; Use different URLs depending on the compression type.  This
          ;; guarantees that /nar URLs remain valid even when 'guix publish'
          ;; is restarted with different compression parameters.

          ;; /nar/gzip/<store-item>
          ((components ... "gzip" store-item)
           (if (and (nar-path? components) (zlib-available?))
               (let ((compression (match compression
                                    (($ <compression> 'gzip)
                                     compression)
                                    (_
                                     %default-gzip-compression))))
                 (if cache
                     (render-nar/cached store cache request store-item
                                        #:compression compression)
                     (render-nar store request store-item
                                 #:compression compression)))
               (not-found request)))

          ;; /nar/<store-item>
          ((components ... store-item)
           (if (nar-path? components)
               (if cache
                   (render-nar/cached store cache request store-item
                                      #:compression %no-compression)
                   (render-nar store request store-item
                               #:compression %no-compression))
               (not-found request)))

          (x (not-found request)))
        (not-found request))))

(define* (run-publish-server socket store
                             #:key (compression %no-compression)
                             (nar-path "nar") narinfo-ttl
                             cache pool)
  (run-server (make-request-handler store
                                    #:cache cache
                                    #:pool pool
                                    #:nar-path nar-path
                                    #:narinfo-ttl narinfo-ttl
                                    #:compression compression)
              concurrent-http-server
              `(#:socket ,socket)))

(define (open-server-socket address)
  "Return a TCP socket bound to ADDRESS, a socket address."
  (let ((sock (socket (sockaddr:fam address) SOCK_STREAM 0)))
    (setsockopt sock SOL_SOCKET SO_REUSEADDR 1)
    (bind sock address)
    sock))

(define (gather-user-privileges user)
  "Switch to the identity of USER, a user name."
  (catch 'misc-error
    (lambda ()
      (let ((user (getpw user)))
        (setgroups #())
        (setgid (passwd:gid user))
        (setuid (passwd:uid user))))
    (lambda (key proc message args . rest)
      (leave (G_ "user '~a' not found: ~a~%")
             user (apply format #f message args)))))

;;;
;;; Entry point.
;;;

(define (guix-publish . args)
  (with-error-handling
    (let* ((opts    (args-fold* args %options
                                (lambda (opt name arg result)
                                  (leave (G_ "~A: unrecognized option~%") name))
                                (lambda (arg result)
                                  (leave (G_ "~A: extraneous argument~%") arg))
                                %default-options))
           (user    (assoc-ref opts 'user))
           (port    (assoc-ref opts 'port))
           (ttl     (assoc-ref opts 'narinfo-ttl))
           (compression (assoc-ref opts 'compression))
           (address (let ((addr (assoc-ref opts 'address)))
                      (make-socket-address (sockaddr:fam addr)
                                           (sockaddr:addr addr)
                                           port)))
           (socket  (open-server-socket address))
           (nar-path  (assoc-ref opts 'nar-path))
           (repl-port (assoc-ref opts 'repl))
           (cache     (assoc-ref opts 'cache))
           (workers   (assoc-ref opts 'workers))

           ;; Read the key right away so that (1) we fail early on if we can't
           ;; access them, and (2) we can then drop privileges.
           (public-key  (read-file-sexp (assoc-ref opts 'public-key-file)))
           (private-key (read-file-sexp (assoc-ref opts 'private-key-file))))

      (when user
        ;; Now that we've read the key material and opened the socket, we can
        ;; drop privileges.
        (gather-user-privileges user))

      (when (zero? (getuid))
        (warning (G_ "server running as root; \
consider using the '--user' option!~%")))

      (parameterize ((%public-key public-key)
                     (%private-key private-key))
        (format #t (G_ "publishing ~a on ~a, port ~d~%")
                %store-directory
                (inet-ntop (sockaddr:fam address) (sockaddr:addr address))
                (sockaddr:port address))
        (when repl-port
          (repl:spawn-server (repl:make-tcp-server-socket #:port repl-port)))

        ;; Set the name of the main thread.
        (set-thread-name "guix publish")

        (with-store store
          (run-publish-server socket store
                              #:cache cache
                              #:pool (and cache (make-pool workers
                                                           #:thread-name
                                                           "publish worker"))
                              #:nar-path nar-path
                              #:compression compression
                              #:narinfo-ttl ttl))))))

;;; Local Variables:
;;; eval: (put 'single-baker 'scheme-indent-function 1)
;;; End: