Features and code reorganization to support the KBase Lakehouse

.github/workflows/autotest_prs.yml

@@ -35,7 +35,7 @@ jobs:
         staticcheck ./...
 
     - name: Set up MinIO
-      uses: infleet/minio-action@v0.0.1
+      uses: cohere-llc/minio-action@v0.0.3
       with:
         port: "9000"
         version: "latest"

.github/workflows/irods.yml

@@ -26,7 +26,7 @@ jobs:
       uses: actions/checkout@v4
 
     - name: Set up MinIO
-      uses: infleet/minio-action@v0.0.1
+      uses: cohere-llc/minio-action@v0.0.3
       with:
         port: "9000"
         version: "latest"

README.md

@@ -45,7 +45,7 @@ require a Minio test instance to be running. You can start one with docker
 or podman:
 
 ```
-docker run -d -p 9000:9000 -p 9001:9001 -e "MINIO_ROOT_USER=minioadmin" -e "MINIO_ROOT_PASSWORD=minioadmin" minio/minio server /data --console-address ":9001"
+docker run -d -p 9000:9000 -p 9001:9001 -e "MINIO_ROOT_USER=minioadmin" -e "MINIO_ROOT_PASSWORD=minioadmin" pgsty/minio server /data --console-address ":9001"
 ```
 
 Then you can run these tests as you would any other Go project:
@@ -87,4 +87,4 @@ to do:
   authenticate with the JGI Data Portal
 
 Alternatively, you can run tests against mock services without the above
-environment variables set by setting `DTS_TEST_WITH_MOCK_SERVICES=true`
+environment variables set by setting `DTS_TEST_WITH_MOCK_SERVICES=true`

auth/auth.go

@@ -33,10 +33,14 @@ type User struct {
 	Organization string
 	// true if this user is a Superuser
 	IsSuper bool
+	// credentials for connections between endpoints with different providers (e.g. Globus <--> S3)
+	ConnectionCredentials map[string]Credential
 }
 
 // A credential used for authorization and authentication
 type Credential struct {
+	// the username associated with this credential
+	Username string `yaml:"username"`
 	// the ID used for authorization (username or UUID)
 	Id string `yaml:"id"`
 	// the secret used for authentication (e.g. password)

auth/authenticator.go

@@ -143,11 +143,12 @@ func (a *Authenticator) readAccessTokenFile() error {
 		}
 
 		userRecords[token] = User{
-			Name:         record[0],
-			Email:        record[1],
-			Orcid:        record[2],
-			Organization: record[3],
-			IsSuper:      isSuper,
+			Name:                  record[0],
+			Email:                 record[1],
+			Orcid:                 record[2],
+			Organization:          record[3],
+			IsSuper:               isSuper,
+			ConnectionCredentials: make(map[string]Credential),
 		}
 	}
 

auth/kbase_auth_server.go

@@ -72,10 +72,10 @@ func NewKBaseAuthServer(accessToken string, options ...KBaseAuthServerOption) (*
 	}
 
 	// check our list of KBase auth server instances for this access token
-	if instances == nil {
-		instances = make(map[string]*KBaseAuthServer)
+	if instances_ == nil {
+		instances_ = make(map[string]*KBaseAuthServer)
 	}
-	if server, found := instances[accessToken]; found {
+	if server, found := instances_[accessToken]; found {
 		return server, nil
 	} else {
 		server := KBaseAuthServer{
@@ -91,7 +91,7 @@ func NewKBaseAuthServer(accessToken string, options ...KBaseAuthServerOption) (*
 		}
 
 		// register this instance of the auth server
-		instances[accessToken] = &server
+		instances_[accessToken] = &server
 		return &server, err
 	}
 }
@@ -103,8 +103,9 @@ func (server KBaseAuthServer) User() (User, error) {
 		return User{}, err
 	}
 	user := User{
-		Name:  kbUser.Display,
-		Email: kbUser.Email,
+		Name:                  kbUser.Display,
+		Email:                 kbUser.Email,
+		ConnectionCredentials: make(map[string]Credential),
 	}
 	for _, pid := range kbUser.Idents {
 		// grab the first ORCID associated with the user
@@ -113,6 +114,23 @@ func (server KBaseAuthServer) User() (User, error) {
 			break
 		}
 	}
+
+	// try to access the MMS in case we're talking to the KBase Lakehouse
+	mms := NewMMS()
+	record, err := mms.FetchRecord(server.AccessToken)
+	if err == nil {
+		user.ConnectionCredentials["s3"] = Credential{
+			Username: record.Username,
+			Id:       record.S3AccessKey,
+			Secret:   record.S3SecretKey,
+		}
+		user.ConnectionCredentials["polaris"] = Credential{
+			Username: record.Username,
+			Id:       record.PolarisClientId,
+			Secret:   record.PolarisClientSecret,
+		}
+	}
+
 	return user, nil
 }
 
@@ -155,7 +173,7 @@ type kbaseAuthErrorResponse struct {
 
 // here's a set of instances to the KBase auth server, mapped by OAuth2
 // access token
-var instances map[string]*KBaseAuthServer
+var instances_ map[string]*KBaseAuthServer
 
 // emits an error representing the error in a response to the auth server
 func kbaseAuthError(response *http.Response) error {

auth/kbase_mms.go

@@ -0,0 +1,85 @@
+// Copyright (c) 2023 The KBase Project and its Contributors
+// Copyright (c) 2023 Cohere Consulting, LLC
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy of
+// this software and associated documentation files (the "Software"), to deal in
+// the Software without restriction, including without limitation the rights to
+// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+// of the Software, and to permit persons to whom the Software is furnished to do
+// so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+package auth
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+)
+
+// The Minio Management Service (MMS) provide authentication information for a user
+// given a valid KBase token for that user
+
+type MMSRecord struct {
+	Username            string `json:"username"`
+	S3AccessKey         string `json:"s3_access_key"`
+	S3SecretKey         string `json:"s3_secret_key"`
+	PolarisClientId     string `json:"polaris_client_id"`
+	PolarisClientSecret string `json:"polaris_client_secret"`
+}
+
+type MMS struct {
+	Client http.Client
+}
+
+func NewMMS() MMS {
+	return MMS{
+		Client: http.Client{
+			Timeout: 5 * time.Second,
+		},
+	}
+}
+
+// retrieves the MMS record associated with the given access token
+func (mms MMS) FetchRecord(accessToken string) (MMSRecord, error) {
+	resource := fmt.Sprintf("%s:%d", kbaseMMSUrl, kbaseMMSPort) + "/credentials/"
+	request, err := http.NewRequest(http.MethodGet, resource, http.NoBody)
+	if err != nil {
+		return MMSRecord{}, err
+	}
+	request.Header.Add("Authorization", fmt.Sprintf("Bearer %s", accessToken))
+	resp, err := mms.Client.Do(request)
+	if err != nil {
+		return MMSRecord{}, err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
+		return MMSRecord{}, fmt.Errorf("MMS returned HTTP %d", resp.StatusCode)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return MMSRecord{}, err
+	}
+
+	var record MMSRecord
+	err = json.Unmarshal(body, &record)
+	return record, err
+}
+
+const (
+	kbaseMMSUrl  = "http://mms.dev"
+	kbaseMMSPort = 8000
+)

databases/kbase/database.go

@@ -52,7 +52,7 @@ func NewDatabase(conf Config) (databases.Database, error) {
 		EndpointName: conf.Endpoint,
 	}
 	var err error
-	db.kbaseFed, err = newKBaseUserFederation(conf.KBaseUserFederationConfig)
+	db.kbaseFed, err = NewKBaseUserFederation(conf.KBaseUserFederationConfig)
 	if err != nil {
 		return nil, err
 	}
@@ -107,7 +107,7 @@ func (db *Database) Finalize(orcid string, id uuid.UUID) error {
 }
 
 func (db *Database) LocalUser(orcid string) (string, error) {
-	return db.kbaseFed.usernameForOrcid(orcid)
+	return db.kbaseFed.UsernameForOrcid(orcid)
 }
 
 func (db Database) Save() (databases.DatabaseSaveState, error) {