code:
;;; GNU Guix --- Functional package management for GNU
;;; Copyright © 2016, 2017, 2018 Ludovic Courtès <ludo@gnu.org>
;;;
;;; This file is part of GNU Guix.
;;;
;;; GNU Guix is free software; you can redistribute it and/or modify it
;;; under the terms of the GNU General Public License as published by
;;; the Free Software Foundation; either version 3 of the License, or (at
;;; your option) any later version.
;;;
;;; GNU Guix is distributed in the hope that it will be useful, but
;;; WITHOUT ANY WARRANTY; without even the implied warranty of
;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
;;; GNU General Public License for more details.
;;;
;;; You should have received a copy of the GNU General Public License
;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.

(define-module (guix ssh)
  #:use-module (guix store)
  #:use-module (guix i18n)
  #:use-module ((guix utils) #:select (&fix-hint))
  #:use-module (ssh session)
  #:use-module (ssh auth)
  #:use-module (ssh key)
  #:use-module (ssh channel)
  #:use-module (ssh popen)
  #:use-module (ssh session)
  #:use-module (ssh dist)
  #:use-module (ssh dist node)
  #:use-module (srfi srfi-1)
  #:use-module (srfi srfi-11)
  #:use-module (srfi srfi-26)
  #:use-module (srfi srfi-34)
  #:use-module (srfi srfi-35)
  #:use-module (ice-9 match)
  #:use-module (ice-9 binary-ports)
  #:export (open-ssh-session
            remote-daemon-channel
            connect-to-remote-daemon
            send-files
            retrieve-files
            retrieve-files*
            remote-store-host

            report-guile-error
            report-module-error))

;;; Commentary:
;;;
;;; This module provides tools to support communication with remote stores
;;; over SSH, using Guile-SSH.
;;;
;;; Code:

(define %compression
  "zlib@openssh.com,zlib")

(define* (open-ssh-session host #:key user port
                           (compression %compression))
  "Open an SSH session for HOST and return it.  When USER and PORT are #f, use
default values or whatever '~/.ssh/config' specifies; otherwise use them.
Throw an error on failure."
  (let ((session (make-session #:user user
                               #:host host
                               #:port port
                               #:timeout 10       ;seconds
                               ;; #:log-verbosity 'protocol

                               ;; We need lightweight compression when
                               ;; exchanging full archives.
                               #:compression compression
                               #:compression-level 3)))

    ;; Honor ~/.ssh/config.
    (session-parse-config! session)

    (match (connect! session)
      ('ok
       ;; Use public key authentication, via the SSH agent if it's available.
       (match (userauth-public-key/auto! session)
         ('success
          session)
         (x
          (disconnect! session)
          (raise (condition
                  (&message
                   (message (format #f (G_ "SSH authentication failed for '~a': ~a~%")
                                    host (get-error session)))))))))
      (x
       ;; Connection failed or timeout expired.
       (raise (condition
               (&message
                (message (format #f (G_ "SSH connection to '~a' failed: ~a~%")
                                 host (get-error session))))))))))

(define* (remote-daemon-channel session
                                #:optional
                                (socket-name
                                 "/var/guix/daemon-socket/socket"))
  "Return an input/output port (an SSH channel) to the daemon at SESSION."
  (define redirect
    ;; Code run in SESSION to redirect the remote process' stdin/stdout to the
    ;; daemon's socket, à la socat.  The SSH protocol supports forwarding to
    ;; Unix-domain sockets but libssh doesn't have an API for that, hence this
    ;; hack.
    `(begin
       (use-modules (ice-9 match) (rnrs io ports)
                    (rnrs bytevectors))

       (let ((sock    (socket AF_UNIX SOCK_STREAM 0))
             (stdin   (current-input-port))
             (stdout  (current-output-port))
             (select* (lambda (read write except)
                        ;; This is a workaround for
                        ;; <https://bugs.gnu.org/30365> in Guile < 2.2.4:
                        ;; since 'select' sometimes returns non-empty sets for
                        ;; no good reason, call 'select' a second time with a
                        ;; zero timeout to filter out incorrect replies.
                        (match (select read write except)
                          ((read write except)
                           (select read write except 0))))))
         (setvbuf stdout _IONBF)

         ;; Use buffered ports so that 'get-bytevector-some' returns up to the
         ;; whole buffer like read(2) would--see <https://bugs.gnu.org/30066>.
         (setvbuf stdin _IOFBF 65536)
         (setvbuf sock _IOFBF 65536)

         (connect sock AF_UNIX ,socket-name)

         (let loop ()
           (match (select* (list stdin sock) '() '())
             ((reads () ())
              (when (memq stdin reads)
                (match (get-bytevector-some stdin)
                  ((? eof-object?)
                   (primitive-exit 0))
                  (bv
                   (put-bytevector sock bv)
                   (force-output sock))))
              (when (memq sock reads)
                (match (get-bytevector-some sock)
                  ((? eof-object?)
                   (primitive-exit 0))
                  (bv
                   (put-bytevector stdout bv))))
              (loop))
             (_
              (primitive-exit 1)))))))

  (open-remote-pipe* session OPEN_BOTH
                     ;; Sort-of shell-quote REDIRECT.
                     "guile" "-c"
                     (object->string
                      (object->string redirect))))

(define* (connect-to-remote-daemon session
                                   #:optional
                                   (socket-name
                                    "/var/guix/daemon-socket/socket"))
  "Connect to the remote build daemon listening on SOCKET-NAME over SESSION,
an SSH session.  Return a <nix-server> object."
  (open-connection #:port (remote-daemon-channel session)))

(define (store-import-channel session)
  "Return an output port to which archives to be exported to SESSION's store
can be written."
  ;; Using the 'import-paths' RPC on a remote store would be slow because it
  ;; makes a round trip every time 32 KiB have been transferred.  This
  ;; procedure instead opens a separate channel to use the remote
  ;; 'import-paths' procedure, which consumes all the data in a single round
  ;; trip.  This optimizes the successful case at the expense of error
  ;; conditions: errors can only be reported once all the input has been
  ;; consumed.
  (define import
    `(begin
       (use-modules (guix) (srfi srfi-34)
                    (rnrs io ports) (rnrs bytevectors))

       (define (consume-input port)
         (let ((bv (make-bytevector 32768)))
           (let loop ()
             (let ((n (get-bytevector-n! port bv 0
                                         (bytevector-length bv))))
               (unless (eof-object? n)
                 (loop))))))

       ;; Upon completion, write an sexp that denotes the status.
       (write
        (catch #t
          (lambda ()
            (guard (c ((nix-protocol-error? c)
                       ;; Consume all the input since the only time we can
                       ;; report the error is after everything has been
                       ;; consumed.
                       (consume-input (current-input-port))
                       (list 'protocol-error (nix-protocol-error-message c))))
              (with-store store
                (setvbuf (current-input-port) _IONBF)
                (import-paths store (current-input-port))
                '(success))))
          (lambda args
            (cons 'error args))))))

  (open-remote-pipe session
                    (string-join
                     `("guile" "-c"
                       ,(object->string (object->string import))))
                    OPEN_BOTH))

(define* (store-export-channel session files
                               #:key recursive?)
  "Return an input port from which an export of FILES from SESSION's store can
be read.  When RECURSIVE? is true, the closure of FILES is exported."
  ;; Same as above: this is more efficient than calling 'export-paths' on a
  ;; remote store.
  (define export
    `(begin
       (eval-when (load expand eval)
         (unless (resolve-module '(guix) #:ensure #f)
           (write `(module-error))
           (exit 7)))

       (use-modules (guix) (srfi srfi-1)
                    (srfi srfi-26) (srfi srfi-34))

       (guard (c ((nix-connection-error? c)
                  (write `(connection-error ,(nix-connection-error-file c)
                                            ,(nix-connection-error-code c))))
                 ((nix-protocol-error? c)
                  (write `(protocol-error ,(nix-protocol-error-status c)
                                          ,(nix-protocol-error-message c))))
                 (else
                  (write `(exception))))
         (with-store store
           (let* ((files ',files)
                  (invalid (remove (cut valid-path? store <>)
                                   files)))
             (unless (null? invalid)
               (write `(invalid-items ,invalid))
               (exit 1))

             ;; TODO: When RECURSIVE? is true, we could send the list of store
             ;; items in the closure so that the other end can filter out
             ;; those it already has.

             (write '(exporting))                 ;we're ready
             (force-output)

             (setvbuf (current-output-port) _IONBF)
             (export-paths store files (current-output-port)
                           #:recursive? ,recursive?))))))

  (open-remote-input-pipe session
                          (string-join
                           `("guile" "-c"
                             ,(object->string
                               (object->string export))))))

(define* (send-files local files remote
                     #:key
                     recursive?
                     (log-port (current-error-port)))
  "Send the subset of FILES from LOCAL (a local store) that's missing to
REMOTE, a remote store.  When RECURSIVE? is true, send the closure of FILES.
Return the list of store items actually sent."
  ;; Compute the subset of FILES missing on SESSION and send them.
  (let* ((files   (if recursive? (requisites local files) files))
         (session (channel-get-session (nix-server-socket remote)))
         (node    (make-node session))
         (missing (node-eval node
                             `(begin
                                (use-modules (guix)
                                             (srfi srfi-1) (srfi srfi-26))

                                (with-store store
                                  (remove (cut valid-path? store <>)
                                          ',files)))))
         (count   (length missing))
         (sizes   (map (lambda (item)
                         (path-info-nar-size (query-path-info local item)))
                       missing))
         (port    (store-import-channel session)))
    (format log-port (N_ "sending ~a store item (~h MiB) to '~a'...~%"
                         "sending ~a store items (~h MiB) to '~a'...~%" count)
            count
            (inexact->exact (round (/ (reduce + 0 sizes) (expt 2. 20))))
            (session-get session 'host))

    ;; Send MISSING in topological order.
    (export-paths local missing port)

    ;; Tell the remote process that we're done.  (In theory the end-of-archive
    ;; mark of 'export-paths' would be enough, but in practice it's not.)
    (channel-send-eof port)

    ;; Wait for completion of the remote process and read the status sexp from
    ;; PORT.
    (let* ((result (false-if-exception (read port)))
           (status (zero? (channel-get-exit-status port))))
      (close-port port)
      (match result
        (('success . _)
         missing)
        (('protocol-error message)
         (raise (condition
                 (&nix-protocol-error (message message) (status 42)))))
        (('error key args ...)
         (raise (condition
                 (&nix-protocol-error
                  (message (call-with-output-string
                             (lambda (port)
                               (print-exception port #f key args))))
                  (status 43)))))
        (_
         (raise (condition
                 (&nix-protocol-error
                  (message "unknown error while sending files over SSH")
                  (status 44)))))))))

(define (remote-store-session remote)
  "Return the SSH channel beneath REMOTE, a remote store as returned by
'connect-to-remote-daemon', or #f."
  (channel-get-session (nix-server-socket remote)))

(define (remote-store-host remote)
  "Return the name of the host REMOTE is connected to, where REMOTE is a
remote store as returned by 'connect-to-remote-daemon'."
  (match (remote-store-session remote)
    (#f #f)
    ((? session? session)
     (session-get session 'host))))

(define* (file-retrieval-port files remote
                              #:key recursive?)
  "Return an input port from which to retrieve FILES (a list of store items)
from REMOTE, along with the number of items to retrieve (lower than or equal
to the length of FILES.)"
  (values (store-export-channel (remote-store-session remote) files
                                #:recursive? recursive?)
          (length files)))            ;XXX: inaccurate when RECURSIVE? is true

(define-syntax raise-error
  (syntax-rules (=>)
    ((_ fmt args ... (=> hint-fmt hint-args ...))
     (raise (condition
             (&message
              (message (format #f fmt args ...)))
             (&fix-hint
              (hint (format #f hint-fmt hint-args ...))))))
    ((_ fmt args ...)
     (raise (condition
             (&message
              (message (format #f fmt args ...))))))))

(define* (retrieve-files* files remote
                          #:key recursive? (log-port (current-error-port))
                          (import (const #f)))
  "Pass IMPORT an input port from which to read the sequence of FILES coming
from REMOTE.  When RECURSIVE? is true, retrieve the closure of FILES."
  (let-values (((port count)
                (file-retrieval-port files remote
                                     #:recursive? recursive?)))
    (match (read port)                            ;read the initial status
      (('exporting)
       (format #t (N_ "retrieving ~a store item from '~a'...~%"
                      "retrieving ~a store items from '~a'...~%" count)
               count (remote-store-host remote))

       (dynamic-wind
         (const #t)
         (lambda ()
           (import port))
         (lambda ()
           (close-port port))))
      ((? eof-object?)
       (report-guile-error (remote-store-host remote)))
      (('module-error . _)
       (report-module-error (remote-store-host remote)))
      (('connection-error file code . _)
       (raise-error (G_ "failed to connect to '~A' on remote host '~A': ~a")
                    file (remote-store-host remote) (strerror code)))
      (('invalid-items items . _)
       (raise-error (N_ "no such item on remote host '~A':~{ ~a~}"
                        "no such items on remote host '~A':~{ ~a~}"
                        (length items))
                    (remote-store-host remote) items))
      (('protocol-error status message . _)
       (raise-error (G_ "protocol error on remote host '~A': ~a")
                    (remote-store-host remote) message))
      (_
       (raise-error (G_ "failed to retrieve store items from '~a'")
                    (remote-store-host remote))))))

(define* (retrieve-files local files remote
                         #:key recursive? (log-port (current-error-port)))
  "Retrieve FILES from REMOTE and import them using the 'import-paths' RPC on
LOCAL.  When RECURSIVE? is true, retrieve the closure of FILES."
  (retrieve-files* (remove (cut valid-path? local <>) files)
                   remote
                   #:recursive? recursive?
                   #:log-port log-port
                   #:import (lambda (port)
                              (import-paths local port))))

;;;
;;; Error reporting.
;;;

(define (report-guile-error host)
  (raise-error (G_ "failed to start Guile on remote host '~A'") host
               (=> (G_ "Make sure @command{guile} can be found in
@code{$PATH} on the remote host.  Run @command{ssh ~A guile --version} to
check.")
                   host)))

(define (report-module-error host)
  "Report an error about missing Guix modules on HOST."
  ;; TRANSLATORS: Leave "Guile" untranslated.
  (raise-error (G_ "Guile modules not found on remote host '~A'") host
               (=> (G_ "Make sure @code{GUILE_LOAD_PATH} includes Guix'
own module directory.  Run @command{ssh ~A env | grep GUILE_LOAD_PATH} to
check.")
                   host)))

;;; ssh.scm ends here


code:
;;; GNU Guix --- Functional package management for GNU
;;; Copyright © 2017, 2018 Ludovic Courtès <ludo@gnu.org>
;;;
;;; This file is part of GNU Guix.
;;;
;;; GNU Guix is free software; you can redistribute it and/or modify it
;;; under the terms of the GNU General Public License as published by
;;; the Free Software Foundation; either version 3 of the License, or (at
;;; your option) any later version.
;;;
;;; GNU Guix is distributed in the hope that it will be useful, but
;;; WITHOUT ANY WARRANTY; without even the implied warranty of
;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
;;; GNU General Public License for more details.
;;;
;;; You should have received a copy of the GNU General Public License
;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.

(define-module (guix status)
  #:use-module (guix records)
  #:use-module (guix i18n)
  #:use-module ((guix ui) #:select (colorize-string))
  #:use-module (guix progress)
  #:autoload   (guix build syscalls) (terminal-columns)
  #:use-module ((guix build download)
                #:select (nar-uri-abbreviation))
  #:use-module (guix store)
  #:use-module (guix derivations)
  #:use-module (srfi srfi-1)
  #:use-module (srfi srfi-9)
  #:use-module (srfi srfi-19)
  #:use-module (srfi srfi-26)
  #:use-module (ice-9 regex)
  #:use-module (ice-9 match)
  #:use-module (ice-9 format)
  #:use-module (ice-9 binary-ports)
  #:autoload   (ice-9 rdelim) (read-string)
  #:use-module (rnrs bytevectors)
  #:use-module ((system foreign)
                #:select (bytevector->pointer pointer->bytevector))
  #:export (build-event-output-port
            compute-status

            build-status
            build-status?
            build-status-building
            build-status-downloading
            build-status-builds-completed
            build-status-downloads-completed

            download?
            download
            download-item
            download-uri
            download-size
            download-start
            download-end
            download-transferred

            build-status-updater
            print-build-event
            print-build-event/quiet
            print-build-status

            with-status-report))

;;; Commentary:
;;;
;;; This module provides facilities to track the status of ongoing builds and
;;; downloads in a given session, as well as tools to report about the current
;;; status to user interfaces.  It does so by analyzing the output of
;;; 'current-build-output-port'.  The build status is maintained in a
;;; <build-status> record.
;;;
;;; Code:

;;;
;;; Build status tracking.
;;;

;; Builds and substitutions performed by the daemon.
(define-record-type* <build-status> build-status make-build-status
  build-status?
  (building     build-status-building             ;list of drv
                (default '()))
  (downloading  build-status-downloading          ;list of <download>
                (default '()))
  (builds-completed build-status-builds-completed ;list of drv
                    (default '()))
  (downloads-completed build-status-downloads-completed ;list of store items
                       (default '())))

;; On-going or completed downloads.  Downloads can be stem from substitutes
;; and from "builtin:download" fixed-output derivations.
(define-record-type <download>
  (%download item uri size start end transferred)
  download?
  (item         download-item)            ;store item
  (uri          download-uri)             ;string | #f
  (size         download-size)            ;integer | #f
  (start        download-start)           ;<time>
  (end          download-end)             ;#f | <time>
  (transferred  download-transferred))    ;integer

(define* (download item uri
                   #:key size
                   (start (current-time time-monotonic)) end
                   (transferred 0))
  "Return a new download."
  (%download item uri size start end transferred))

(define (matching-download item)
  "Return a predicate that matches downloads of ITEM."
  (lambda (download)
    (string=? item (download-item download))))

(define* (compute-status event status
                         #:key (current-time current-time))
  "Given EVENT, a tuple like (build-started \"/gnu/store/...-foo.drv\" ...),
compute a new status based on STATUS."
  (match event
    (('build-started drv _ ...)
     (build-status
      (inherit status)
      (building (cons drv (build-status-building status)))))
    (((or 'build-succeeded 'build-failed) drv _ ...)
     (build-status
      (inherit status)
      (building (delete drv (build-status-building status)))
      (builds-completed (cons drv (build-status-builds-completed status)))))

    ;; Note: Ignore 'substituter-started' and 'substituter-succeeded' because
    ;; they're not as informative as 'download-started' and
    ;; 'download-succeeded'.

    (('download-started item uri (= string->number size))
     ;; This is presumably a fixed-output derivation so move it from
     ;; 'building' to 'downloading'.  XXX: This doesn't work in 'check' mode
     ;; because ITEM is different from DRV's output.
     (build-status
      (inherit status)
      (building (remove (lambda (drv)
                          (equal? (false-if-exception
                                   (derivation->output-path
                                    (read-derivation-from-file drv)))
                                  item))
                        (build-status-building status)))
      (downloading (cons (download item uri #:size size
                                   #:start (current-time time-monotonic))
                         (build-status-downloading status)))))
    (('download-succeeded item uri (= string->number size))
     (let ((current (find (matching-download item)
                          (build-status-downloading status))))
       (build-status
        (inherit status)
        (downloading (delq current (build-status-downloading status)))
        (downloads-completed
         (cons (download item uri
                         #:size size
                         #:start (download-start current)
                         #:transferred size
                         #:end (current-time time-monotonic))
               (build-status-downloads-completed status))))))
    (('substituter-succeeded item _ ...)
     (match (find (matching-download item)
                  (build-status-downloading status))
       (#f
        ;; Presumably we already got a 'download-succeeded' event for ITEM,
        ;; everything is fine.
        status)
       (current
        ;; Maybe the build process didn't emit a 'download-succeeded' event
        ;; for ITEM, so remove CURRENT from the queue now.
        (build-status
         (inherit status)
         (downloading (delq current (build-status-downloading status)))
         (downloads-completed
          (cons (download item (download-uri current)
                          #:size (download-size current)
                          #:start (download-start current)
                          #:transferred (download-size current)
                          #:end (current-time time-monotonic))
                (build-status-downloads-completed status)))))))
    (('download-progress item uri
                         (= string->number size)
                         (= string->number transferred))
     (let ((downloads (remove (matching-download item)
                              (build-status-downloading status)))
           (current   (find (matching-download item)
                            (build-status-downloading status))))
       (build-status
        (inherit status)
        (downloading (cons (download item uri
                                     #:size size
                                     #:start
                                     (or (and current
                                              (download-start current))
                                         (current-time time-monotonic))
                                     #:transferred transferred)
                           downloads)))))
    (_
     status)))

(define (simultaneous-jobs status)
  "Return the number of on-going builds and downloads for STATUS."
  (+ (length (build-status-building status))
     (length (build-status-downloading status))))

;;;
;;; Rendering.
;;;

(define (extended-build-trace-supported?)
  "Return true if the currently used store is known to support \"extended
build traces\" such as \"@ download-progress\" traces."
  ;; Support for extended build traces was added in protocol version #x162.
  (and (current-store-protocol-version)
       (>= (current-store-protocol-version) #x162)))

(define spin!
  (let ((steps (circular-list "\\" "|" "/" "-")))
    (lambda (port)
      "Display a spinner on PORT."
      (match steps
        ((first . rest)
         (set! steps rest)
         (display "\r\x1b[K" port)
         (display first port)
         (force-output port))))))

(define (color-output? port)
  "Return true if we should write colored output to PORT."
  (and (not (getenv "INSIDE_EMACS"))
       (not (getenv "NO_COLOR"))
       (isatty? port)))

(define-syntax color-rules
  (syntax-rules ()
    "Return a procedure that colorizes the string it is passed according to
the given rules.  Each rule has the form:

  (REGEXP COLOR1 COLOR2 ...)

where COLOR1 specifies how to colorize the first submatch of REGEXP, and so
on."
    ((_ (regexp colors ...) rest ...)
     (let ((next (color-rules rest ...))
           (rx   (make-regexp regexp)))
       (lambda (str)
         (if (string-index str #\nul)
             str
             (match (regexp-exec rx str)
               (#f (next str))
               (m  (let loop ((n 1)
                              (c '(colors ...))
                              (result '()))
                     (match c
                       (()
                        (string-concatenate-reverse result))
                       ((first . tail)
                        (loop (+ n 1) tail
                              (cons (colorize-string (match:substring m n)
                                                     first)
                                    result)))))))))))
    ((_)
     (lambda (str)
       str))))

(define colorize-log-line
  ;; Take a string and return a possibly colorized string according to the
  ;; rules below.
  (color-rules
   ("^(phase)(.*)(succeeded after)(.*)(seconds)(.*)"
    GREEN    BOLD GREEN          RESET  GREEN  BLUE)
   ("^(phase)(.*)(failed after)(.*)(seconds)(.*)"
    RED BLUE RED BLUE RED BLUE)
   ("^(.*)(error|fail|failed|\\<FAIL|FAILED)([[:blank:]]*)(:)(.*)"
    RESET  RED                           BOLD         BOLD BOLD)
   ("^(.*)(warning)([[:blank:]]*)(:)(.*)"
    RESET  MAGENTA   BOLD        BOLD BOLD)))

(define* (print-build-event event old-status status
                            #:optional (port (current-error-port))
                            #:key
                            (colorize? (color-output? port))
                            (print-log? #t))
  "Print information about EVENT and STATUS to PORT.  When COLORIZE? is true,
produce colorful output.  When PRINT-LOG? is true, display the build log in
addition to build events."
  (define info
    (if colorize?
        (cut colorize-string <> 'BOLD)
        identity))

  (define success
    (if colorize?
        (cut colorize-string <> 'GREEN 'BOLD)
        identity))

  (define failure
    (if colorize?
        (cut colorize-string <> 'RED 'BOLD)
        identity))

  (define print-log-line
    (if print-log?
        (if colorize?
            (lambda (line)
              (display (colorize-log-line line) port))
            (cut display <> port))
        (lambda (line)
          (spin! port))))

  (display "\r" port)                             ;erase the spinner
  (match event
    (('build-started drv . _)
     (format port (info (G_ "building ~a...")) drv)
     (newline port))
    (('build-succeeded drv . _)
     (when (or print-log? (not (extended-build-trace-supported?)))
       (format port (success (G_ "successfully built ~a")) drv)
       (newline port))
     (match (build-status-building status)
       (() #t)
       (ongoing                                   ;when max-jobs > 1
        (format port
                (N_ "The following build is still in progress:~%~{  ~a~%~}~%"
                    "The following builds are still in progress:~%~{  ~a~%~}~%"
                    (length ongoing))
                ongoing))))
    (('build-failed drv . _)
     (format port (failure (G_ "build of ~a failed")) drv)
     (newline port)
     (match (derivation-log-file drv)
       (#f
        (format port (failure (G_ "Could not find build log for '~a'."))
                drv))
       (log
        (format port (info (G_ "View build log at '~a'.")) log)))
     (newline port))
    (('substituter-started item _ ...)
     (when (or print-log? (not (extended-build-trace-supported?)))
       (format port (info (G_ "substituting ~a...")) item)
       (newline port)))
    (('download-started item uri _ ...)
     (format port (info (G_ "downloading from ~a...")) uri)
     (newline port))
    (('download-progress item uri
                         (= string->number size)
                         (= string->number transferred))
     ;; Print a progress bar, but only if there's only one on-going
     ;; job--otherwise the output would be intermingled.
     (when (= 1 (simultaneous-jobs status))
       (match (find (matching-download item)
                    (build-status-downloading status))
         (#f #f)                                  ;shouldn't happen!
         (download
          ;; XXX: It would be nice to memoize the abbreviation.
          (let ((uri (if (string-contains uri "/nar/")
                         (nar-uri-abbreviation uri)
                         (basename uri))))
            (display-download-progress uri size
                                       #:start-time
                                       (download-start download)
                                       #:transferred transferred))))))
    (('substituter-succeeded item _ ...)
     ;; If there are no jobs running, we already reported download completion
     ;; so there's nothing left to do.
     (unless (and (zero? (simultaneous-jobs status))
                  (extended-build-trace-supported?))
       (format port (success (G_ "substitution of ~a complete")) item)
       (newline port)))
    (('substituter-failed item _ ...)
     (format port (failure (G_ "substitution of ~a failed")) item)
     (newline port))
    (('hash-mismatch item algo expected actual _ ...)
     ;; TRANSLATORS: The final string looks like "sha256 hash mismatch for
     ;; /gnu/store/…-sth:", where "sha256" is the hash algorithm.
     (format port (failure (G_ "~a hash mismatch for ~a:")) algo item)
     (newline port)
     (format port (info (G_ "\
  expected hash: ~a
  actual hash:   ~a~%"))
             expected actual))
    (('build-log line)
     ;; TODO: Better distinguish daemon messages and build log lines.
     (cond ((string-prefix? "substitute: " line)
            ;; The daemon prefixes early messages coming with 'guix
            ;; substitute' with "substitute:".  These are useful ("updating
            ;; substitutes from URL"), so let them through.
            (format port line)
            (force-output port))
           ((string-prefix? "waiting for locks" line)
            ;; This is when a derivation is already being built and we're just
            ;; waiting for the build to complete.
            (display (info (string-trim-right line)) port)
            (newline))
           (else
            (print-log-line line))))
    (_
     event)))

(define* (print-build-event/quiet event old-status status
                                  #:optional
                                  (port (current-error-port))
                                  #:key
                                  (colorize? (color-output? port)))
  (print-build-event event old-status status port
                     #:colorize? colorize?
                     #:print-log? #f))

(define* (build-status-updater #:optional (on-change (const #t)))
  "Return a procedure that can be passed to 'build-event-output-port'.  That
procedure computes the new build status upon each event and calls ON-CHANGE:

  (ON-CHANGE event status new-status)

ON-CHANGE can display the build status, build events, etc."
  (lambda (event status)
    (let ((new (compute-status event status)))
      (on-change event status new)
      new)))

;;;
;;; Build port.
;;;

(define %newline
  (char-set #\return #\newline))

(define (maybe-utf8->string bv)
  "Attempt to decode BV as UTF-8 string and return it.  Gracefully handle the
case where BV does not contain only valid UTF-8."
  (catch 'decoding-error
    (lambda ()
      (utf8->string bv))
    (lambda _
      ;; This is the sledgehammer but it's the only safe way we have to
      ;; properly handle this.  It's expensive but it's rarely needed.
      (let ((port (open-bytevector-input-port bv)))
        (set-port-encoding! port "UTF-8")
        (set-port-conversion-strategy! port 'substitute)
        (let ((str (read-string port)))
          (close-port port)
          str)))))

(define* (build-event-output-port proc #:optional (seed (build-status)))
  "Return an output port for use as 'current-build-output-port' that calls
PROC with its current state value, initialized with SEED, on every build
event.  Build events passed to PROC are tuples corresponding to the \"build
traces\" produced by the daemon:

  (build-started \"/gnu/store/...-foo.drv\" ...)
  (substituter-started \"/gnu/store/...-foo\" ...)

and so on.

The second return value is a thunk to retrieve the current state."
  (define %fragments
    ;; Line fragments received so far.
    '())

  (define %state
    ;; Current state for PROC.
    seed)

  (define (process-line line)
    (if (string-prefix? "@ " line)
        (match (string-tokenize (string-drop line 2))
          (((= string->symbol event-name) args ...)
           (set! %state
             (proc (cons event-name args)
                   %state))))
        (set! %state (proc (list 'build-log line)
                           %state))))

  (define (bytevector-range bv offset count)
    (let ((ptr (bytevector->pointer bv offset)))
      (pointer->bytevector ptr count)))

  (define (write! bv offset count)
    (let loop ((str (maybe-utf8->string (bytevector-range bv offset count))))
      (match (string-index str %newline)
        ((? integer? cr)
         (let ((tail (string-take str (+ 1 cr))))
           (process-line (string-concatenate-reverse
                          (cons tail %fragments)))
           (set! %fragments '())
           (loop (string-drop str (+ 1 cr)))))
        (#f
         (unless (string-null? str)
           (set! %fragments (cons str %fragments)))
         count))))

  (define port
    (make-custom-binary-output-port "filtering-input-port"
                                    write!
                                    #f #f
                                    #f))

  ;; The build port actually receives Unicode strings.
  (set-port-encoding! port "UTF-8")
  (setvbuf port (cond-expand (guile-2.2 'line) (else _IOLBF)))

  (values port (lambda () %state)))

(define (call-with-status-report on-event thunk)
  (parameterize ((current-terminal-columns (terminal-columns))
                 (current-build-output-port
                  (build-event-output-port (build-status-updater on-event))))
    (thunk)))

(define-syntax-rule (with-status-report on-event exp ...)
  "Set up build status reporting to the user using the ON-EVENT procedure;
evaluate EXP... in that context."
  (call-with-status-report on-event (lambda () exp ...)))

license: mit