diff --git a/data/datasource_aws_sm.go b/data/datasource_aws_sm.go index f513d060..e6c100ed 100644 --- a/data/datasource_aws_sm.go +++ b/data/datasource_aws_sm.go @@ -1,7 +1,10 @@ package data import ( + "fmt" + "net/url" "path" + "strings" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/secretsmanager" @@ -15,16 +18,41 @@ type awsSecretsManagerGetter interface { GetSecretValue(input *secretsmanager.GetSecretValueInput) (*secretsmanager.GetSecretValueOutput, error) } -func parseAWSSecretsManagerArgs(origPath string, args ...string) (paramPath string, err error) { - paramPath = origPath - if len(args) >= 1 { - paramPath = path.Join(paramPath, args[0]) +func parseDatasourceURLArgs(sourceURL *url.URL, args ...string) (params map[string]interface{}, p string, err error) { + if len(args) >= 2 { + err = fmt.Errorf("maximum two arguments to %s datasource: alias, extraPath (found %d)", + sourceURL.Scheme, len(args)) + return nil, "", err } - if len(args) >= 2 { - err = errors.New("Maximum two arguments to aws+sm datasource: alias, extraPath") + p = sourceURL.Path + params = make(map[string]interface{}) + for key, val := range sourceURL.Query() { + params[key] = strings.Join(val, " ") + } + + if p == "" && sourceURL.Opaque != "" { + p = sourceURL.Opaque + } + + if len(args) == 1 { + parsed, err := url.Parse(args[0]) + if err != nil { + return nil, "", err + } + + if parsed.Path != "" { + p = path.Join(p, parsed.Path) + if strings.HasSuffix(parsed.Path, "/") { + p += "/" + } + } + + for key, val := range parsed.Query() { + params[key] = strings.Join(val, " ") + } } - return + return params, p, nil } func readAWSSecretsManager(source *Source, args ...string) (output []byte, err error) { @@ -32,7 +60,7 @@ func readAWSSecretsManager(source *Source, args ...string) (output []byte, err e source.awsSecretsManager = secretsmanager.New(gaws.SDKSession()) } - paramPath, err := parseAWSSecretsManagerArgs(source.URL.Path, args...) + _, paramPath, err := parseDatasourceURLArgs(source.URL, args...) if err != nil { return nil, err } diff --git a/data/datasource_aws_sm_test.go b/data/datasource_aws_sm_test.go index 0b88d156..3ed0aa45 100644 --- a/data/datasource_aws_sm_test.go +++ b/data/datasource_aws_sm_test.go @@ -41,27 +41,57 @@ func simpleAWSSecretsManagerSourceHelper(dummyGetter awsSecretsManagerGetter) *S } } -func TestAWSSecretsManager_ParseArgsSimple(t *testing.T) { - paramPath, err := parseAWSSecretsManagerArgs("noddy") - assert.Equal(t, "noddy", paramPath) - assert.Nil(t, err) -} - -func TestAWSSecretsManager_ParseArgsAppend(t *testing.T) { - paramPath, err := parseAWSSecretsManagerArgs("base", "extra") - assert.Equal(t, "base/extra", paramPath) - assert.Nil(t, err) -} +func TestAWSSecretsManager_ParseAWSSecretsManagerArgs(t *testing.T) { + _, _, err := parseDatasourceURLArgs(mustParseURL("base"), "extra", "too many!") + assert.Error(t, err) -func TestAWSSecretsManager_ParseArgsAppend2(t *testing.T) { - paramPath, err := parseAWSSecretsManagerArgs("/foo/", "/extra") - assert.Equal(t, "/foo/extra", paramPath) - assert.Nil(t, err) -} + data := []struct { + u *url.URL + args []string + expectedParams map[string]interface{} + expectedPath string + }{ + {mustParseURL("noddy"), nil, nil, "noddy"}, + {mustParseURL("base"), []string{"extra"}, nil, "base/extra"}, + {mustParseURL("/foo/"), []string{"/extra"}, nil, "/foo/extra"}, + {mustParseURL("aws+sm:///foo"), []string{"bar"}, nil, "/foo/bar"}, + {mustParseURL("aws+sm:foo"), nil, nil, "foo"}, + {mustParseURL("aws+sm:foo/bar"), nil, nil, "foo/bar"}, + {mustParseURL("aws+sm:/foo/bar"), nil, nil, "/foo/bar"}, + {mustParseURL("aws+sm:foo"), []string{"baz"}, nil, "foo/baz"}, + {mustParseURL("aws+sm:foo/bar"), []string{"baz"}, nil, "foo/bar/baz"}, + {mustParseURL("aws+sm:/foo/bar"), []string{"baz"}, nil, "/foo/bar/baz"}, + {mustParseURL("aws+sm:///foo"), []string{"dir/"}, nil, "/foo/dir/"}, + {mustParseURL("aws+sm:///foo/"), nil, nil, "/foo/"}, + {mustParseURL("aws+sm:///foo/"), []string{"baz"}, nil, "/foo/baz"}, + + {mustParseURL("aws+sm:foo?type=text/plain"), []string{"baz"}, + map[string]interface{}{"type": "text/plain"}, "foo/baz"}, + {mustParseURL("aws+sm:foo/bar?type=text/plain"), []string{"baz"}, + map[string]interface{}{"type": "text/plain"}, "foo/bar/baz"}, + {mustParseURL("aws+sm:/foo/bar?type=text/plain"), []string{"baz"}, + map[string]interface{}{"type": "text/plain"}, "/foo/bar/baz"}, + { + mustParseURL("aws+sm:/foo/bar?type=text/plain"), + []string{"baz/qux?type=application/json¶m=quux"}, + map[string]interface{}{ + "type": "application/json", + "param": "quux", + }, + "/foo/bar/baz/qux", + }, + } -func TestAWSSecretsManager_ParseArgsTooMany(t *testing.T) { - _, err := parseAWSSecretsManagerArgs("base", "extra", "too many!") - assert.Error(t, err) + for _, d := range data { + params, p, err := parseDatasourceURLArgs(d.u, d.args...) + assert.NoError(t, err) + if d.expectedParams == nil { + assert.Empty(t, params) + } else { + assert.EqualValues(t, d.expectedParams, params) + } + assert.Equal(t, d.expectedPath, p) + } } func TestAWSSecretsManager_GetParameterSetup(t *testing.T) { diff --git a/data/datasource_awssmp.go b/data/datasource_awssmp.go index c03c1897..26d93855 100644 --- a/data/datasource_awssmp.go +++ b/data/datasource_awssmp.go @@ -2,8 +2,6 @@ package data import ( "context" - "net/url" - "path" "strings" "github.com/aws/aws-sdk-go/aws" @@ -20,42 +18,13 @@ type awssmpGetter interface { GetParametersByPathWithContext(ctx context.Context, input *ssm.GetParametersByPathInput, opts ...request.Option) (*ssm.GetParametersByPathOutput, error) } -func parseAWSSMPArgs(sourceURL *url.URL, args ...string) (params map[string]interface{}, p string, err error) { - if len(args) >= 2 { - err = errors.New("Maximum two arguments to aws+smp datasource: alias, extraPath") - return nil, "", err - } - - p = sourceURL.Path - params = make(map[string]interface{}) - for key, val := range sourceURL.Query() { - params[key] = strings.Join(val, " ") - } - - if len(args) == 1 { - parsed, err := url.Parse(args[0]) - if err != nil { - return nil, "", err - } - - if parsed.Path != "" { - p = path.Join(p, parsed.Path) - } - - for key, val := range parsed.Query() { - params[key] = strings.Join(val, " ") - } - } - return params, p, err -} - func readAWSSMP(source *Source, args ...string) (data []byte, err error) { ctx := context.TODO() if source.asmpg == nil { source.asmpg = ssm.New(gaws.SDKSession()) } - _, paramPath, err := parseAWSSMPArgs(source.URL, args...) + _, paramPath, err := parseDatasourceURLArgs(source.URL, args...) if err != nil { return nil, err } diff --git a/data/datasource_awssmp_test.go b/data/datasource_awssmp_test.go index 35602641..bb0a03c8 100644 --- a/data/datasource_awssmp_test.go +++ b/data/datasource_awssmp_test.go @@ -57,33 +57,6 @@ func simpleAWSSourceHelper(dummy awssmpGetter) *Source { } } -func TestAWSSMP_ParseArgsSimple(t *testing.T) { - u, _ := url.Parse("noddy") - _, p, err := parseAWSSMPArgs(u) - assert.Equal(t, "noddy", p) - assert.Nil(t, err) -} - -func TestAWSSMP_ParseArgsAppend(t *testing.T) { - u, _ := url.Parse("base") - _, p, err := parseAWSSMPArgs(u, "extra") - assert.Equal(t, "base/extra", p) - assert.Nil(t, err) -} - -func TestAWSSMP_ParseArgsAppend2(t *testing.T) { - u, _ := url.Parse("/foo/") - _, p, err := parseAWSSMPArgs(u, "/extra") - assert.Equal(t, "/foo/extra", p) - assert.Nil(t, err) -} - -func TestAWSSMP_ParseArgsTooMany(t *testing.T) { - u, _ := url.Parse("base") - _, _, err := parseAWSSMPArgs(u, "extra", "too many!") - assert.Error(t, err) -} - func TestAWSSMP_GetParameterSetup(t *testing.T) { calledOk := false s := simpleAWSSourceHelper(DummyParamGetter{ diff --git a/data/datasource_env.go b/data/datasource_env.go index 34b2e7d5..15d56a1d 100644 --- a/data/datasource_env.go +++ b/data/datasource_env.go @@ -9,8 +9,7 @@ import ( func readEnv(source *Source, args ...string) (b []byte, err error) { n := source.URL.Path n = strings.TrimPrefix(n, "/") - if n != "" { - } else if n == "" { + if n == "" { n = source.URL.Opaque } diff --git a/data/datasource_vault.go b/data/datasource_vault.go index 2ed7d9f8..6f3fe142 100644 --- a/data/datasource_vault.go +++ b/data/datasource_vault.go @@ -1,7 +1,6 @@ package data import ( - "net/url" "strings" "github.com/pkg/errors" @@ -9,30 +8,6 @@ import ( "github.com/hairyhenderson/gomplate/v3/vault" ) -func parseVaultParams(sourceURL *url.URL, args []string) (params map[string]interface{}, p string, err error) { - p = sourceURL.Path - params = make(map[string]interface{}) - for key, val := range sourceURL.Query() { - params[key] = strings.Join(val, " ") - } - - if len(args) == 1 { - parsed, err := url.Parse(args[0]) - if err != nil { - return nil, "", err - } - - if parsed.Path != "" { - p = p + "/" + parsed.Path - } - - for key, val := range parsed.Query() { - params[key] = strings.Join(val, " ") - } - } - return params, p, nil -} - func readVault(source *Source, args ...string) (data []byte, err error) { if source.vc == nil { source.vc, err = vault.New(source.URL) @@ -45,7 +20,7 @@ func readVault(source *Source, args ...string) (data []byte, err error) { } } - params, p, err := parseVaultParams(source.URL, args) + params, p, err := parseDatasourceURLArgs(source.URL, args...) if err != nil { return nil, err } diff --git a/docs/content/datasources.md b/docs/content/datasources.md index 1d1e9778..67959b25 100644 --- a/docs/content/datasources.md +++ b/docs/content/datasources.md @@ -35,7 +35,7 @@ For our purposes, the _scheme_ and the _path_ components are especially importan ### Opaque URIs -For some more advanced datasources, such as the [`merge` scheme](#using-merge-datasources), opaque URIs are used (rather than a hierarchical URL): +For some datasources, such as the [`merge`](#using-merge-datasources), [`aws+sm`](#using-aws-sm-datasources), and [`aws+smp`](#using-aws-smp-datasources) schemes, opaque URIs can be used (rather than a hierarchical URL): ```pre scheme path query fragment @@ -160,10 +160,10 @@ See details on how to configure gomplate's AWS support in [_Configuring AWS_](.. ### URL Considerations -The _scheme_ and _path_ URL components are used by this datasource. +The _scheme_ and _path_ URL components are used by this datasource. This may be an _opaque_ URI instead of an URL, when the key does not begin with a `/` character (e.g. `aws+smp:myparam`). - the _scheme_ must be `aws+smp` -- the _path_ component is used to specify the path to the parameter. [Directory](#directory-datasources) semantics are available when the path ends with a `/` character. +- the _path_ component is used to specify the path to the parameter (this may be a hierarchical path beginning with `/`, or an opaque path). [Directory](#directory-datasources) semantics are available when the path ends with a `/` character. ### Output @@ -186,6 +186,7 @@ Given your [AWS account's Parameter Store](https://eu-west-1.console.aws.amazon. - `/foo/first/others` - `Bill,Ben` (a StringList) - `/foo/first/password` - `super-secret` (a SecureString) - `/foo/second/p1` - `aaa` +- `myparameter` - `bar` ```console $ echo '{{ ds "foo" }}' | gomplate -d foo=aws+smp:///foo/first/password @@ -205,12 +206,19 @@ $ gomplate -d foo=aws+smp:///foo/first/ -i '{{ range (ds "foo") }} {{- end }}' others: Bill,Ben password: super-secret + +$ gomplate -d foo=aws+smp:myparameter -i '{{ (ds "foo").Value }} +bar ``` ## Using `aws+sm` datasource ### URL Considerations -For `aws+sm`, only the _scheme_ and _path_ components are necessary to be defined. Other URL components are ignored. + +For `aws+sm`, only the _scheme_ and _path_ components are necessary to be defined. This may be an _opaque_ URI instead of an URL, when the key does not begin with a `/` character (e.g. `aws+sm:myparam`). + +- the _scheme_ must be `aws+sm` +- the _path_ component is used to specify the path to the secret (this may be a hierarchical path beginning with `/`, or an opaque path) ### Output @@ -221,6 +229,7 @@ The output will be the SecretString from the `GetSecretValueOutput` object from Given your [AWS account's Secret Manager](https://eu-central-1.console.aws.amazon.com/secretsmanager/home?region=eu-central-1#/listSecrets) has the following data: - `/foo/bar/password` - `super-secret` +- `mysecret` - `bar` ```console $ echo '{{ (ds "foo") }}' | gomplate -d foo=aws+sm:///foo/bar/password @@ -231,6 +240,9 @@ super-secret $ echo '{{ (ds "foo" "/bar/password") }}' | gomplate -d foo=aws+sm:///foo/ super-secret + +$ echo '{{ (ds "foo") }}' | gomplate -d foo=aws+sm:mysecret +bar ``` ## Using `s3` datasources