const fs = require(`fs-extra`)
const got = require(`got`)
const crypto = require(`crypto`)
const path = require(`path`)
const { isWebUri } = require(`valid-url`)

const { createFileNode } = require(`./create-file-node`)
const cacheId = url => `create-remote-file-node-${url}`

module.exports = ({ url, store, cache, createNode, auth = {} }) =>
  new Promise(async (resolve, reject) => {
    if (!url || isWebUri(url) === undefined) {
      resolve()
      return
    }

    // Ensure our cache directory exists.
    await fs.ensureDir(
      path.join(
        store.getState().program.directory,
        `.cache`,
        `gatsby-source-filesystem`
      )
    )

    // See if there's response headers for this url
    // from a previous request.
    const cachedHeaders = await cache.get(cacheId(url))
    const headers = {
      auth: auth.htaccess_user + `:` + auth.htaccess_pass,
    }
    if (cachedHeaders && cachedHeaders.etag) {
      headers[`If-None-Match`] = cachedHeaders.etag
    }

    // Create the temp and permanent file names for the url.
    const digest = crypto
      .createHash(`md5`)
      .update(url)
      .digest(`hex`)
    const tmpFilename = path.join(
      store.getState().program.directory,
      `.cache`,
      `gatsby-source-filesystem`,
      `tmp-` + digest + path.parse(url).ext
    )
    const filename = path.join(
      store.getState().program.directory,
      `.cache`,
      `gatsby-source-filesystem`,
      digest + path.parse(url).ext
    )

    // Fetch the file.
    let statusCode
    let responseHeaders
    let responseError = false
    const responseStream = got.stream(url, headers)
    responseStream.pipe(fs.createWriteStream(tmpFilename))
    responseStream.on(`downloadProgress`, pro => console.log(pro))

    // If there's a 400/500 response or other error.
    responseStream.on(`error`, (error, body, response) => {
      responseError = true
      fs.removeSync(tmpFilename)
      reject(error, body, response)
    })

    // If the status code is 200, move the piped temp file to the real name.
    // Else if 304, remove the empty response.
    responseStream.on(`response`, response => {
      statusCode = response.statusCode
      responseHeaders = response.headers
    })

    responseStream.on(`end`, response => {
      if (responseError) return

      // Save the response headers for future requests.
      cache.set(cacheId(url), responseHeaders)
      if (statusCode === 200) {
        fs.moveSync(tmpFilename, filename, { overwrite: true })
      } else {
        fs.removeSync(tmpFilename)
      }

      // Create the file node and return.
      createFileNode(filename, {}).then(fileNode => {
        // Override the default plugin as gatsby-source-filesystem needs to
        // be the owner of File nodes or there'll be conflicts if any other
        // File nodes are created through normal usages of
        // gatsby-source-filesystem.
        createNode(fileNode, { name: `gatsby-source-filesystem` })
        resolve(fileNode)
      })
    })
  })

@@ -823,48 +823,257 @@
  = %7B
+%7D%0A
 %0A    
-  auth: auth.htaccess_user + %60:%60 + 
+// Add htaccess authentication if passed in. This isn't particularly%0A    // extensible. We should define a proper API that we validate.%0A    if (auth && auth.htaccess_pass && auth.htaccess_user) %7B%0A      headers.auth = %60$%7Bauth.htaccess_user%7D:$%7B
 auth
@@ -1090,16 +1090,18 @@
 pass
-,
+%7D%60
 %0A    %7D%0A
+%0A
     

Improve checks on authentication so have wiggle room in future