Merge pull request #874 from hairyhenderson/aws-sm-parse-opaque-urls-868

Allow referencing aws+sm[p] keys that don't start with '/'

data/datasource_aws_sm.go

@@ -1,7 +1,10 @@
 package data
 
 import (
+	"fmt"
+	"net/url"
 	"path"
+	"strings"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/secretsmanager"
@@ -15,24 +18,49 @@ type awsSecretsManagerGetter interface {
 	GetSecretValue(input *secretsmanager.GetSecretValueInput) (*secretsmanager.GetSecretValueOutput, error)
 }
 
-func parseAWSSecretsManagerArgs(origPath string, args ...string) (paramPath string, err error) {
-	paramPath = origPath
-	if len(args) >= 1 {
-		paramPath = path.Join(paramPath, args[0])
+func parseDatasourceURLArgs(sourceURL *url.URL, args ...string) (params map[string]interface{}, p string, err error) {
+	if len(args) >= 2 {
+		err = fmt.Errorf("maximum two arguments to %s datasource: alias, extraPath (found %d)",
+			sourceURL.Scheme, len(args))
+		return nil, "", err
 	}
 
-	if len(args) >= 2 {
-		err = errors.New("Maximum two arguments to aws+sm datasource: alias, extraPath")
+	p = sourceURL.Path
+	params = make(map[string]interface{})
+	for key, val := range sourceURL.Query() {
+		params[key] = strings.Join(val, " ")
+	}
+
+	if p == "" && sourceURL.Opaque != "" {
+		p = sourceURL.Opaque
+	}
+
+	if len(args) == 1 {
+		parsed, err := url.Parse(args[0])
+		if err != nil {
+			return nil, "", err
+		}
+
+		if parsed.Path != "" {
+			p = path.Join(p, parsed.Path)
+			if strings.HasSuffix(parsed.Path, "/") {
+				p += "/"
+			}
+		}
+
+		for key, val := range parsed.Query() {
+			params[key] = strings.Join(val, " ")
+		}
 	}
-	return
+	return params, p, nil
 }
 
 func readAWSSecretsManager(source *Source, args ...string) (output []byte, err error) {
 	if source.awsSecretsManager == nil {
 		source.awsSecretsManager = secretsmanager.New(gaws.SDKSession())
 	}
 
-	paramPath, err := parseAWSSecretsManagerArgs(source.URL.Path, args...)
+	_, paramPath, err := parseDatasourceURLArgs(source.URL, args...)
 	if err != nil {
 		return nil, err
 	}

data/datasource_aws_sm_test.go

@@ -41,27 +41,57 @@ func simpleAWSSecretsManagerSourceHelper(dummyGetter awsSecretsManagerGetter) *S
 	}
 }
 
-func TestAWSSecretsManager_ParseArgsSimple(t *testing.T) {
-	paramPath, err := parseAWSSecretsManagerArgs("noddy")
-	assert.Equal(t, "noddy", paramPath)
-	assert.Nil(t, err)
-}
-
-func TestAWSSecretsManager_ParseArgsAppend(t *testing.T) {
-	paramPath, err := parseAWSSecretsManagerArgs("base", "extra")
-	assert.Equal(t, "base/extra", paramPath)
-	assert.Nil(t, err)
-}
+func TestAWSSecretsManager_ParseAWSSecretsManagerArgs(t *testing.T) {
+	_, _, err := parseDatasourceURLArgs(mustParseURL("base"), "extra", "too many!")
+	assert.Error(t, err)
 
-func TestAWSSecretsManager_ParseArgsAppend2(t *testing.T) {
-	paramPath, err := parseAWSSecretsManagerArgs("/foo/", "/extra")
-	assert.Equal(t, "/foo/extra", paramPath)
-	assert.Nil(t, err)
-}
+	data := []struct {
+		u              *url.URL
+		args           []string
+		expectedParams map[string]interface{}
+		expectedPath   string
+	}{
+		{mustParseURL("noddy"), nil, nil, "noddy"},
+		{mustParseURL("base"), []string{"extra"}, nil, "base/extra"},
+		{mustParseURL("/foo/"), []string{"/extra"}, nil, "/foo/extra"},
+		{mustParseURL("aws+sm:///foo"), []string{"bar"}, nil, "/foo/bar"},
+		{mustParseURL("aws+sm:foo"), nil, nil, "foo"},
+		{mustParseURL("aws+sm:foo/bar"), nil, nil, "foo/bar"},
+		{mustParseURL("aws+sm:/foo/bar"), nil, nil, "/foo/bar"},
+		{mustParseURL("aws+sm:foo"), []string{"baz"}, nil, "foo/baz"},
+		{mustParseURL("aws+sm:foo/bar"), []string{"baz"}, nil, "foo/bar/baz"},
+		{mustParseURL("aws+sm:/foo/bar"), []string{"baz"}, nil, "/foo/bar/baz"},
+		{mustParseURL("aws+sm:///foo"), []string{"dir/"}, nil, "/foo/dir/"},
+		{mustParseURL("aws+sm:///foo/"), nil, nil, "/foo/"},
+		{mustParseURL("aws+sm:///foo/"), []string{"baz"}, nil, "/foo/baz"},
+
+		{mustParseURL("aws+sm:foo?type=text/plain"), []string{"baz"},
+			map[string]interface{}{"type": "text/plain"}, "foo/baz"},
+		{mustParseURL("aws+sm:foo/bar?type=text/plain"), []string{"baz"},
+			map[string]interface{}{"type": "text/plain"}, "foo/bar/baz"},
+		{mustParseURL("aws+sm:/foo/bar?type=text/plain"), []string{"baz"},
+			map[string]interface{}{"type": "text/plain"}, "/foo/bar/baz"},
+		{
+			mustParseURL("aws+sm:/foo/bar?type=text/plain"),
+			[]string{"baz/qux?type=application/json&param=quux"},
+			map[string]interface{}{
+				"type":  "application/json",
+				"param": "quux",
+			},
+			"/foo/bar/baz/qux",
+		},
+	}
 
-func TestAWSSecretsManager_ParseArgsTooMany(t *testing.T) {
-	_, err := parseAWSSecretsManagerArgs("base", "extra", "too many!")
-	assert.Error(t, err)
+	for _, d := range data {
+		params, p, err := parseDatasourceURLArgs(d.u, d.args...)
+		assert.NoError(t, err)
+		if d.expectedParams == nil {
+			assert.Empty(t, params)
+		} else {
+			assert.EqualValues(t, d.expectedParams, params)
+		}
+		assert.Equal(t, d.expectedPath, p)
+	}
 }
 
 func TestAWSSecretsManager_GetParameterSetup(t *testing.T) {

data/datasource_awssmp.go

@@ -2,8 +2,6 @@ package data
 
 import (
 	"context"
-	"net/url"
-	"path"
 	"strings"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -20,42 +18,13 @@ type awssmpGetter interface {
 	GetParametersByPathWithContext(ctx context.Context, input *ssm.GetParametersByPathInput, opts ...request.Option) (*ssm.GetParametersByPathOutput, error)
 }
 
-func parseAWSSMPArgs(sourceURL *url.URL, args ...string) (params map[string]interface{}, p string, err error) {
-	if len(args) >= 2 {
-		err = errors.New("Maximum two arguments to aws+smp datasource: alias, extraPath")
-		return nil, "", err
-	}
-
-	p = sourceURL.Path
-	params = make(map[string]interface{})
-	for key, val := range sourceURL.Query() {
-		params[key] = strings.Join(val, " ")
-	}
-
-	if len(args) == 1 {
-		parsed, err := url.Parse(args[0])
-		if err != nil {
-			return nil, "", err
-		}
-
-		if parsed.Path != "" {
-			p = path.Join(p, parsed.Path)
-		}
-
-		for key, val := range parsed.Query() {
-			params[key] = strings.Join(val, " ")
-		}
-	}
-	return params, p, err
-}
-
 func readAWSSMP(source *Source, args ...string) (data []byte, err error) {
 	ctx := context.TODO()
 	if source.asmpg == nil {
 		source.asmpg = ssm.New(gaws.SDKSession())
 	}
 
-	_, paramPath, err := parseAWSSMPArgs(source.URL, args...)
+	_, paramPath, err := parseDatasourceURLArgs(source.URL, args...)
 	if err != nil {
 		return nil, err
 	}

data/datasource_awssmp_test.go

@@ -57,33 +57,6 @@ func simpleAWSSourceHelper(dummy awssmpGetter) *Source {
 	}
 }
 
-func TestAWSSMP_ParseArgsSimple(t *testing.T) {
-	u, _ := url.Parse("noddy")
-	_, p, err := parseAWSSMPArgs(u)
-	assert.Equal(t, "noddy", p)
-	assert.Nil(t, err)
-}
-
-func TestAWSSMP_ParseArgsAppend(t *testing.T) {
-	u, _ := url.Parse("base")
-	_, p, err := parseAWSSMPArgs(u, "extra")
-	assert.Equal(t, "base/extra", p)
-	assert.Nil(t, err)
-}
-
-func TestAWSSMP_ParseArgsAppend2(t *testing.T) {
-	u, _ := url.Parse("/foo/")
-	_, p, err := parseAWSSMPArgs(u, "/extra")
-	assert.Equal(t, "/foo/extra", p)
-	assert.Nil(t, err)
-}
-
-func TestAWSSMP_ParseArgsTooMany(t *testing.T) {
-	u, _ := url.Parse("base")
-	_, _, err := parseAWSSMPArgs(u, "extra", "too many!")
-	assert.Error(t, err)
-}
-
 func TestAWSSMP_GetParameterSetup(t *testing.T) {
 	calledOk := false
 	s := simpleAWSSourceHelper(DummyParamGetter{

data/datasource_env.go

@@ -9,8 +9,7 @@ import (
 func readEnv(source *Source, args ...string) (b []byte, err error) {
 	n := source.URL.Path
 	n = strings.TrimPrefix(n, "/")
-	if n != "" {
-	} else if n == "" {
+	if n == "" {
 		n = source.URL.Opaque
 	}
 

data/datasource_vault.go

@@ -1,38 +1,13 @@
 package data
 
 import (
-	"net/url"
 	"strings"
 
 	"github.com/pkg/errors"
 
 	"github.com/hairyhenderson/gomplate/v3/vault"
 )
 
-func parseVaultParams(sourceURL *url.URL, args []string) (params map[string]interface{}, p string, err error) {
-	p = sourceURL.Path
-	params = make(map[string]interface{})
-	for key, val := range sourceURL.Query() {
-		params[key] = strings.Join(val, " ")
-	}
-
-	if len(args) == 1 {
-		parsed, err := url.Parse(args[0])
-		if err != nil {
-			return nil, "", err
-		}
-
-		if parsed.Path != "" {
-			p = p + "/" + parsed.Path
-		}
-
-		for key, val := range parsed.Query() {
-			params[key] = strings.Join(val, " ")
-		}
-	}
-	return params, p, nil
-}
-
 func readVault(source *Source, args ...string) (data []byte, err error) {
 	if source.vc == nil {
 		source.vc, err = vault.New(source.URL)
@@ -45,7 +20,7 @@ func readVault(source *Source, args ...string) (data []byte, err error) {
 		}
 	}
 
-	params, p, err := parseVaultParams(source.URL, args)
+	params, p, err := parseDatasourceURLArgs(source.URL, args...)
 	if err != nil {
 		return nil, err
 	}

docs/content/datasources.md

@@ -35,7 +35,7 @@ For our purposes, the _scheme_ and the _path_ components are especially importan
 
 ### Opaque URIs
 
-For some more advanced datasources, such as the [`merge` scheme](#using-merge-datasources), opaque URIs are used (rather than a hierarchical URL):
+For some datasources, such as the [`merge`](#using-merge-datasources), [`aws+sm`](#using-aws-sm-datasources), and [`aws+smp`](#using-aws-smp-datasources) schemes, opaque URIs can be used (rather than a hierarchical URL):
 
 ```pre
 scheme                   path        query   fragment
@@ -160,10 +160,10 @@ See details on how to configure gomplate's AWS support in [_Configuring AWS_](..
 
 ### URL Considerations
 
-The _scheme_ and _path_ URL components are used by this datasource.
+The _scheme_ and _path_ URL components are used by this datasource. This may be an _opaque_ URI instead of an URL, when the key does not begin with a `/` character (e.g. `aws+smp:myparam`).
 
 - the _scheme_ must be `aws+smp`
-- the _path_ component is used to specify the path to the parameter. [Directory](#directory-datasources) semantics are available when the path ends with a `/` character.
+- the _path_ component is used to specify the path to the parameter (this may be a hierarchical path beginning with `/`, or an opaque path). [Directory](#directory-datasources) semantics are available when the path ends with a `/` character.
 
 ### Output
 
@@ -186,6 +186,7 @@ Given your [AWS account's Parameter Store](https://eu-west-1.console.aws.amazon.
 - `/foo/first/others` - `Bill,Ben` (a StringList)
 - `/foo/first/password` - `super-secret` (a SecureString)
 - `/foo/second/p1` - `aaa`
+- `myparameter` - `bar`
 
 ```console
 $ echo '{{ ds "foo" }}' | gomplate -d foo=aws+smp:///foo/first/password
@@ -205,12 +206,19 @@ $ gomplate -d foo=aws+smp:///foo/first/ -i '{{ range (ds "foo") }}
 {{- end }}'
 others: Bill,Ben
 password: super-secret
+
+$ gomplate -d foo=aws+smp:myparameter -i '{{ (ds "foo").Value }}
+bar
 ```
 
 ## Using `aws+sm` datasource
 
 ### URL Considerations
-For `aws+sm`, only the _scheme_ and _path_ components are necessary to be defined. Other URL components are ignored.
+
+For `aws+sm`, only the _scheme_ and _path_ components are necessary to be defined. This may be an _opaque_ URI instead of an URL, when the key does not begin with a `/` character (e.g. `aws+sm:myparam`).
+
+- the _scheme_ must be `aws+sm`
+- the _path_ component is used to specify the path to the secret (this may be a hierarchical path beginning with `/`, or an opaque path)
 
 ### Output
 
@@ -221,6 +229,7 @@ The output will be the SecretString from the `GetSecretValueOutput` object from
 Given your [AWS account's Secret Manager](https://eu-central-1.console.aws.amazon.com/secretsmanager/home?region=eu-central-1#/listSecrets) has the following data:
 
 - `/foo/bar/password` - `super-secret`
+- `mysecret` - `bar`
 
 ```console
 $ echo '{{ (ds "foo") }}' | gomplate -d foo=aws+sm:///foo/bar/password
@@ -231,6 +240,9 @@ super-secret
 
 $ echo '{{ (ds "foo" "/bar/password") }}' | gomplate -d foo=aws+sm:///foo/
 super-secret
+
+$ echo '{{ (ds "foo") }}' | gomplate -d foo=aws+sm:mysecret
+bar
 ```
 
 ## Using `s3` datasources