2017-06-06 15:58:56 +00:00
|
|
|
package getter
|
|
|
|
|
|
|
|
import (
|
2019-01-24 02:07:01 +00:00
|
|
|
"context"
|
2017-06-06 15:58:56 +00:00
|
|
|
"encoding/xml"
|
|
|
|
"fmt"
|
|
|
|
"io"
|
|
|
|
"net/http"
|
|
|
|
"net/url"
|
|
|
|
"os"
|
|
|
|
"path/filepath"
|
2019-01-24 02:07:01 +00:00
|
|
|
"strconv"
|
2017-06-06 15:58:56 +00:00
|
|
|
"strings"
|
2018-12-20 21:43:52 +00:00
|
|
|
|
2019-01-24 02:07:01 +00:00
|
|
|
safetemp "github.com/hashicorp/go-safetemp"
|
2017-06-06 15:58:56 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
// HttpGetter is a Getter implementation that will download from an HTTP
|
|
|
|
// endpoint.
|
|
|
|
//
|
|
|
|
// For file downloads, HTTP is used directly.
|
|
|
|
//
|
2019-01-24 02:07:01 +00:00
|
|
|
// The protocol for downloading a directory from an HTTP endpoint is as follows:
|
2017-06-06 15:58:56 +00:00
|
|
|
//
|
|
|
|
// An HTTP GET request is made to the URL with the additional GET parameter
|
|
|
|
// "terraform-get=1". This lets you handle that scenario specially if you
|
|
|
|
// wish. The response must be a 2xx.
|
|
|
|
//
|
|
|
|
// First, a header is looked for "X-Terraform-Get" which should contain
|
|
|
|
// a source URL to download.
|
|
|
|
//
|
|
|
|
// If the header is not present, then a meta tag is searched for named
|
|
|
|
// "terraform-get" and the content should be a source URL.
|
|
|
|
//
|
|
|
|
// The source URL, whether from the header or meta tag, must be a fully
|
|
|
|
// formed URL. The shorthand syntax of "github.com/foo/bar" or relative
|
|
|
|
// paths are not allowed.
|
|
|
|
type HttpGetter struct {
|
2019-01-24 02:07:01 +00:00
|
|
|
getter
|
|
|
|
|
2017-06-06 15:58:56 +00:00
|
|
|
// Netrc, if true, will lookup and use auth information found
|
|
|
|
// in the user's netrc file if available.
|
|
|
|
Netrc bool
|
2018-01-10 18:52:15 +00:00
|
|
|
|
|
|
|
// Client is the http.Client to use for Get requests.
|
|
|
|
// This defaults to a cleanhttp.DefaultClient if left unset.
|
|
|
|
Client *http.Client
|
2018-12-20 21:43:52 +00:00
|
|
|
|
|
|
|
// Header contains optional request header fields that should be included
|
|
|
|
// with every HTTP request. Note that the zero value of this field is nil,
|
|
|
|
// and as such it needs to be initialized before use, via something like
|
|
|
|
// make(http.Header).
|
|
|
|
Header http.Header
|
2017-06-06 15:58:56 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (g *HttpGetter) ClientMode(u *url.URL) (ClientMode, error) {
|
|
|
|
if strings.HasSuffix(u.Path, "/") {
|
|
|
|
return ClientModeDir, nil
|
|
|
|
}
|
|
|
|
return ClientModeFile, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (g *HttpGetter) Get(dst string, u *url.URL) error {
|
2019-01-24 02:07:01 +00:00
|
|
|
ctx := g.Context()
|
2017-06-06 15:58:56 +00:00
|
|
|
// Copy the URL so we can modify it
|
|
|
|
var newU url.URL = *u
|
|
|
|
u = &newU
|
|
|
|
|
|
|
|
if g.Netrc {
|
|
|
|
// Add auth from netrc if we can
|
|
|
|
if err := addAuthFromNetrc(u); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-01-10 18:52:15 +00:00
|
|
|
if g.Client == nil {
|
|
|
|
g.Client = httpClient
|
|
|
|
}
|
|
|
|
|
2017-06-06 15:58:56 +00:00
|
|
|
// Add terraform-get to the parameter.
|
|
|
|
q := u.Query()
|
|
|
|
q.Add("terraform-get", "1")
|
|
|
|
u.RawQuery = q.Encode()
|
|
|
|
|
|
|
|
// Get the URL
|
2018-12-20 21:43:52 +00:00
|
|
|
req, err := http.NewRequest("GET", u.String(), nil)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
req.Header = g.Header
|
|
|
|
resp, err := g.Client.Do(req)
|
2017-06-06 15:58:56 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2018-12-20 21:43:52 +00:00
|
|
|
|
2017-06-06 15:58:56 +00:00
|
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
|
|
|
return fmt.Errorf("bad response code: %d", resp.StatusCode)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Extract the source URL
|
|
|
|
var source string
|
|
|
|
if v := resp.Header.Get("X-Terraform-Get"); v != "" {
|
|
|
|
source = v
|
|
|
|
} else {
|
|
|
|
source, err = g.parseMeta(resp.Body)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if source == "" {
|
|
|
|
return fmt.Errorf("no source URL was returned")
|
|
|
|
}
|
|
|
|
|
|
|
|
// If there is a subdir component, then we download the root separately
|
|
|
|
// into a temporary directory, then copy over the proper subdir.
|
|
|
|
source, subDir := SourceDirSubdir(source)
|
|
|
|
if subDir == "" {
|
2019-01-24 02:07:01 +00:00
|
|
|
var opts []ClientOption
|
|
|
|
if g.client != nil {
|
|
|
|
opts = g.client.Options
|
|
|
|
}
|
|
|
|
return Get(dst, source, opts...)
|
2017-06-06 15:58:56 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// We have a subdir, time to jump some hoops
|
2019-01-24 02:07:01 +00:00
|
|
|
return g.getSubdir(ctx, dst, source, subDir)
|
2017-06-06 15:58:56 +00:00
|
|
|
}
|
|
|
|
|
2019-01-24 02:07:01 +00:00
|
|
|
func (g *HttpGetter) GetFile(dst string, src *url.URL) error {
|
|
|
|
ctx := g.Context()
|
2018-01-10 18:52:15 +00:00
|
|
|
if g.Netrc {
|
|
|
|
// Add auth from netrc if we can
|
2019-01-24 02:07:01 +00:00
|
|
|
if err := addAuthFromNetrc(src); err != nil {
|
2018-01-10 18:52:15 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-01-24 02:07:01 +00:00
|
|
|
// Create all the parent directories if needed
|
|
|
|
if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
f, err := os.OpenFile(dst, os.O_RDWR|os.O_CREATE, os.FileMode(0666))
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2019-02-26 23:25:34 +00:00
|
|
|
defer f.Close()
|
2019-01-24 02:07:01 +00:00
|
|
|
|
2018-01-10 18:52:15 +00:00
|
|
|
if g.Client == nil {
|
|
|
|
g.Client = httpClient
|
|
|
|
}
|
|
|
|
|
2019-01-24 02:07:01 +00:00
|
|
|
var currentFileSize int64
|
|
|
|
|
|
|
|
// We first make a HEAD request so we can check
|
|
|
|
// if the server supports range queries. If the server/URL doesn't
|
|
|
|
// support HEAD requests, we just fall back to GET.
|
|
|
|
req, err := http.NewRequest("HEAD", src.String(), nil)
|
2017-06-06 15:58:56 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2019-01-24 02:07:01 +00:00
|
|
|
if g.Header != nil {
|
|
|
|
req.Header = g.Header
|
|
|
|
}
|
|
|
|
headResp, err := g.Client.Do(req)
|
|
|
|
if err == nil && headResp != nil {
|
|
|
|
headResp.Body.Close()
|
|
|
|
if headResp.StatusCode == 200 {
|
|
|
|
// If the HEAD request succeeded, then attempt to set the range
|
|
|
|
// query if we can.
|
|
|
|
if headResp.Header.Get("Accept-Ranges") == "bytes" {
|
|
|
|
if fi, err := f.Stat(); err == nil {
|
|
|
|
if _, err = f.Seek(0, os.SEEK_END); err == nil {
|
|
|
|
req.Header.Set("Range", fmt.Sprintf("bytes=%d-", fi.Size()))
|
|
|
|
currentFileSize = fi.Size()
|
|
|
|
totalFileSize, _ := strconv.ParseInt(headResp.Header.Get("Content-Length"), 10, 64)
|
|
|
|
if currentFileSize >= totalFileSize {
|
|
|
|
// file already present
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
req.Method = "GET"
|
2018-12-20 21:43:52 +00:00
|
|
|
|
|
|
|
resp, err := g.Client.Do(req)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2019-01-24 02:07:01 +00:00
|
|
|
switch resp.StatusCode {
|
|
|
|
case http.StatusOK, http.StatusPartialContent:
|
|
|
|
// all good
|
|
|
|
default:
|
|
|
|
resp.Body.Close()
|
2017-06-06 15:58:56 +00:00
|
|
|
return fmt.Errorf("bad response code: %d", resp.StatusCode)
|
|
|
|
}
|
|
|
|
|
2019-01-24 02:07:01 +00:00
|
|
|
body := resp.Body
|
2017-06-06 15:58:56 +00:00
|
|
|
|
2019-01-24 02:07:01 +00:00
|
|
|
if g.client != nil && g.client.ProgressListener != nil {
|
|
|
|
// track download
|
|
|
|
fn := filepath.Base(src.EscapedPath())
|
|
|
|
body = g.client.ProgressListener.TrackProgress(fn, currentFileSize, currentFileSize+resp.ContentLength, resp.Body)
|
2017-06-06 15:58:56 +00:00
|
|
|
}
|
2019-01-24 02:07:01 +00:00
|
|
|
defer body.Close()
|
2017-06-06 15:58:56 +00:00
|
|
|
|
2019-01-24 02:07:01 +00:00
|
|
|
n, err := Copy(ctx, f, body)
|
2018-12-20 21:43:52 +00:00
|
|
|
if err == nil && n < resp.ContentLength {
|
|
|
|
err = io.ErrShortWrite
|
|
|
|
}
|
2017-06-06 15:58:56 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
// getSubdir downloads the source into the destination, but with
|
|
|
|
// the proper subdir.
|
2019-01-24 02:07:01 +00:00
|
|
|
func (g *HttpGetter) getSubdir(ctx context.Context, dst, source, subDir string) error {
|
2018-12-20 21:43:52 +00:00
|
|
|
// Create a temporary directory to store the full source. This has to be
|
|
|
|
// a non-existent directory.
|
|
|
|
td, tdcloser, err := safetemp.Dir("", "getter")
|
2017-06-06 15:58:56 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2018-12-20 21:43:52 +00:00
|
|
|
defer tdcloser.Close()
|
2018-01-10 18:52:15 +00:00
|
|
|
|
2019-01-24 02:07:01 +00:00
|
|
|
var opts []ClientOption
|
|
|
|
if g.client != nil {
|
|
|
|
opts = g.client.Options
|
|
|
|
}
|
2017-06-06 15:58:56 +00:00
|
|
|
// Download that into the given directory
|
2019-01-24 02:07:01 +00:00
|
|
|
if err := Get(td, source, opts...); err != nil {
|
2017-06-06 15:58:56 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2018-01-10 18:52:15 +00:00
|
|
|
// Process any globbing
|
|
|
|
sourcePath, err := SubdirGlob(td, subDir)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2017-06-06 15:58:56 +00:00
|
|
|
// Make sure the subdir path actually exists
|
|
|
|
if _, err := os.Stat(sourcePath); err != nil {
|
|
|
|
return fmt.Errorf(
|
|
|
|
"Error downloading %s: %s", source, err)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Copy the subdirectory into our actual destination.
|
|
|
|
if err := os.RemoveAll(dst); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
// Make the final destination
|
|
|
|
if err := os.MkdirAll(dst, 0755); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2019-01-24 02:07:01 +00:00
|
|
|
return copyDir(ctx, dst, sourcePath, false)
|
2017-06-06 15:58:56 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// parseMeta looks for the first meta tag in the given reader that
|
|
|
|
// will give us the source URL.
|
|
|
|
func (g *HttpGetter) parseMeta(r io.Reader) (string, error) {
|
|
|
|
d := xml.NewDecoder(r)
|
|
|
|
d.CharsetReader = charsetReader
|
|
|
|
d.Strict = false
|
|
|
|
var err error
|
|
|
|
var t xml.Token
|
|
|
|
for {
|
|
|
|
t, err = d.Token()
|
|
|
|
if err != nil {
|
|
|
|
if err == io.EOF {
|
|
|
|
err = nil
|
|
|
|
}
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
if e, ok := t.(xml.StartElement); ok && strings.EqualFold(e.Name.Local, "body") {
|
|
|
|
return "", nil
|
|
|
|
}
|
|
|
|
if e, ok := t.(xml.EndElement); ok && strings.EqualFold(e.Name.Local, "head") {
|
|
|
|
return "", nil
|
|
|
|
}
|
|
|
|
e, ok := t.(xml.StartElement)
|
|
|
|
if !ok || !strings.EqualFold(e.Name.Local, "meta") {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
if attrValue(e.Attr, "name") != "terraform-get" {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
if f := attrValue(e.Attr, "content"); f != "" {
|
|
|
|
return f, nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// attrValue returns the attribute value for the case-insensitive key
|
|
|
|
// `name', or the empty string if nothing is found.
|
|
|
|
func attrValue(attrs []xml.Attr, name string) string {
|
|
|
|
for _, a := range attrs {
|
|
|
|
if strings.EqualFold(a.Name.Local, name) {
|
|
|
|
return a.Value
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return ""
|
|
|
|
}
|
|
|
|
|
|
|
|
// charsetReader returns a reader for the given charset. Currently
|
|
|
|
// it only supports UTF-8 and ASCII. Otherwise, it returns a meaningful
|
|
|
|
// error which is printed by go get, so the user can find why the package
|
|
|
|
// wasn't downloaded if the encoding is not supported. Note that, in
|
|
|
|
// order to reduce potential errors, ASCII is treated as UTF-8 (i.e. characters
|
|
|
|
// greater than 0x7f are not rejected).
|
|
|
|
func charsetReader(charset string, input io.Reader) (io.Reader, error) {
|
|
|
|
switch strings.ToLower(charset) {
|
|
|
|
case "ascii":
|
|
|
|
return input, nil
|
|
|
|
default:
|
|
|
|
return nil, fmt.Errorf("can't decode XML document using charset %q", charset)
|
|
|
|
}
|
|
|
|
}
|