SM-826: move all configuration to configuration files (#73)

- added embedded configuration
- used shared dimo library for parsing yaml files
- switch dev devices to connect directly to emqx
- fixed scan-vin command

README.md

@@ -97,7 +97,7 @@ Change the template on the device to "no loggers" id 117.
 
 ### Deploying binary to device
 
-- build binary use command at beginning - targeting linux: `GOARCH=arm GOOS=linux go build -ldflags="-s -w -X 'main.Version=v1.0.0'" -o edge-network && upx edge-network`
+- build binary use command at beginning - targeting linux: `GOARCH=arm GOOS=linux go build -ldflags="-s -w -X 'main.Version=v1.0.0' -X 'main.ENV=dev'" -o edge-network && upx edge-network`
 - scp edge-network [email protected]:~
 - ssh [email protected]
 - sudo systemctl stop edge-network

can_dump.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"flag"
 	"github.com/DIMO-Network/edge-network/commands"
+	"github.com/DIMO-Network/edge-network/config"
 	"github.com/DIMO-Network/edge-network/internal/loggers"
 	"github.com/google/subcommands"
 	"github.com/google/uuid"
@@ -43,7 +44,7 @@ func (p *canDumpCmd) SetFlags(f *flag.FlagSet) {
 	f.IntVar(&p.chunkSize, "send", 0, "send result over mqtt to the s3 bucket using <chunk_size>")
 }
 
-func (p *canDumpCmd) Execute(_ context.Context, _ *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus {
+func (p *canDumpCmd) Execute(_ context.Context, _ *flag.FlagSet, args ...interface{}) subcommands.ExitStatus {
 	log := zerolog.New(os.Stdout).With().
 		Timestamp().
 		Str("app", "edge-network").
@@ -76,14 +77,20 @@ func (p *canDumpCmd) Execute(_ context.Context, _ *flag.FlagSet, _ ...interface{
 		return subcommands.ExitFailure
 	}
 
+	// read config file
+	conf, ok := args[0].(config.Config)
+	if !ok {
+		log.Error().Msg("unable to read config file")
+		return subcommands.ExitFailure
+	}
 	if p.chunkSize > 0 && p.save {
-		mqttErr := canDumperInstance.WriteToMQTT(log, unitID, *ethAddr, p.chunkSize, string(currentTime), true)
+		mqttErr := canDumperInstance.WriteToMQTT(log, unitID, *ethAddr, p.chunkSize, string(currentTime), true, conf)
 		if mqttErr != nil {
 			log.Err(mqttErr).Send()
 			return subcommands.ExitFailure
 		}
 	} else if p.chunkSize > 0 {
-		mqttErr := canDumperInstance.WriteToMQTT(log, unitID, *ethAddr, p.chunkSize, string(currentTime), true)
+		mqttErr := canDumperInstance.WriteToMQTT(log, unitID, *ethAddr, p.chunkSize, string(currentTime), true, conf)
 		if mqttErr != nil {
 			log.Err(mqttErr).Send()
 			return subcommands.ExitFailure

certificate/certificate.go

@@ -12,7 +12,7 @@ import (
 	"encoding/pem"
 	"fmt"
 	"github.com/DIMO-Network/edge-network/commands"
-	"github.com/DIMO-Network/edge-network/internal/gateways"
+	"github.com/DIMO-Network/edge-network/config"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/google/uuid"
@@ -31,11 +31,7 @@ import (
 	"time"
 )
 
-const generateChallenge = "/auth/web3/generate_challenge"
-const submitChallenge = "/auth/web3/submit_challenge"
 const domain = "http://127.0.0.1:10000"
-const CertPath = "/opt/autopi/client.crt"
-const PrivateKeyPath = "/opt/autopi/client.pem"
 
 //go:generate mockgen -source certificate.go -destination mocks/certificate_mock.go
 type Signer interface {
@@ -69,48 +65,35 @@ func (r CertFileWriter) IsNotExist(err error) bool {
 }
 
 type Service struct {
-	logger            zerolog.Logger
-	oauthURL          string
-	oauthClientID     string
-	oauthClientSecret string
-	caURL             string
-	caFingerprint     string
-	certificatePath   string
-	stepCa            Signer
-	fileSys           FileSystem
+	logger               zerolog.Logger
+	oauthURL             string
+	oauthClientID        string
+	oauthClientSecret    string
+	caURL                string
+	caFingerprint        string
+	certificatePath      string
+	privateKeyPath       string
+	generateChallengeURI string
+	submitChallengeURI   string
+	stepCa               Signer
+	fileSys              FileSystem
 }
 
-func NewCertificateService(logger zerolog.Logger, env gateways.Environment, client Signer, fileSys FileSystem) *Service {
-	// set the auth and ca urls based on the environment
-	var authURL string
-	var caURL string
-	var oauthClientID string
-	var oauthClientSecret string
-	var caFingerprint string
-	if env == gateways.Development {
-		authURL = "https://auth.dev.dimo.zone"
-		caURL = "https://ca.dev.dimo.zone"
-		oauthClientID = "step-ca"
-		oauthClientSecret = "KsQ7pruHob6D3NLFQEg9"
-		caFingerprint = "a563363f0bc9cc76031695743c059cf1e694f294e4d1548e981d18cb96348f5f"
-	} else {
-		authURL = "https://auth.dimo.zone"
-		caURL = "https://ca.dimo.zone"
-		oauthClientID = "step-ca"
-		oauthClientSecret = "mkoLsNAfiG2DM2DfqYsX"
-		caFingerprint = "9992e3ce6a87c5d8dc6a09daddd4365c9e0f50593f3e897dedc1b89c037270ed"
-	}
-
+func NewCertificateService(logger zerolog.Logger, conf config.Config, client Signer, fileSys FileSystem) *Service {
 	return &Service{
-		logger:            logger,
-		oauthURL:          authURL,
-		oauthClientID:     oauthClientID,
-		oauthClientSecret: oauthClientSecret,
-		caURL:             caURL,
-		caFingerprint:     caFingerprint,
-		certificatePath:   CertPath,
-		stepCa:            client,
-		fileSys:           fileSys,
+		logger:               logger,
+		oauthURL:             conf.Services.Auth.Host,
+		oauthClientID:        conf.Services.Auth.ClientID,
+		oauthClientSecret:    conf.Services.Auth.ClientSecret,
+		caURL:                conf.Services.Ca.Host,
+		caFingerprint:        conf.Services.Auth.CaFingerprint,
+		certificatePath:      conf.Services.Ca.CertPath,
+		privateKeyPath:       conf.Services.Ca.PrivateKeyPath,
+		generateChallengeURI: conf.Services.Auth.GenerateChallengeURI,
+		submitChallengeURI:   conf.Services.Auth.SubmitChallengeURI,
+		// the below are needed mostly for the testing
+		stepCa:  client,
+		fileSys: fileSys,
 	}
 }
 
@@ -214,7 +197,7 @@ func (cs *Service) SignWeb3Certificate(ethAddress string, confirm bool, unitID u
 	pemData := pem.EncodeToMemory(pemBlock)
 
 	// Write the PEM data to a file
-	err = cs.fileSys.WriteFile(PrivateKeyPath, pemData, 0644)
+	err = cs.fileSys.WriteFile(cs.privateKeyPath, pemData, 0644)
 	if err != nil {
 		return "", err
 	}
@@ -247,9 +230,9 @@ func (cs *Service) GetOauthToken(ethAddress string, unitID uuid.UUID) (string, e
 	initParams.Set("scope", "openid email")
 	initParams.Set("address", ethAddress)
 
-	resp, err := http.PostForm(cs.oauthURL+generateChallenge, initParams)
+	resp, err := http.PostForm(cs.oauthURL+cs.generateChallengeURI, initParams)
 	if err != nil {
-		return "", err
+		return "", errors.Wrap(err, fmt.Sprintf("error requesting challenge from oauth server: %+v\n", initParams))
 	}
 	defer resp.Body.Close()
 
@@ -283,21 +266,21 @@ func (cs *Service) GetOauthToken(ethAddress string, unitID uuid.UUID) (string, e
 	submitParams.Set("signature", signedChallenge)
 	submitParams.Set("client_secret", cs.oauthClientSecret)
 
-	resp, err = http.Post(cs.oauthURL+submitChallenge, "application/x-www-form-urlencoded", strings.NewReader(submitParams.Encode()))
+	resp, err = http.Post(cs.oauthURL+cs.submitChallengeURI, "application/x-www-form-urlencoded", strings.NewReader(submitParams.Encode()))
 	if err != nil {
-		return "", err
+		return "", errors.Wrap(err, fmt.Sprintf("error submit challenge to oauth server: %+v\n", submitParams))
 	}
 	defer resp.Body.Close()
 
 	body, err = io.ReadAll(resp.Body)
 	if err != nil {
-		return "", err
+		return "", errors.Wrap(err, fmt.Sprintf("error reading response body: %+v\n", resp.Body))
 	}
 
 	// Extract 'access_token' from the response body
 	var tokenResp TokenResponse
 	if err := json.Unmarshal(body, &tokenResp); err != nil {
-		return "", err
+		return "", errors.Wrap(err, fmt.Sprintf("error unmarshalling response body: %s\n", body))
 	}
 
 	return tokenResp.AccessToken, nil

certificate/certificate_test.go

@@ -7,7 +7,7 @@ import (
 	"crypto/x509"
 	"crypto/x509/pkix"
 	"fmt"
-	"github.com/DIMO-Network/edge-network/internal/gateways"
+	dimoConfig "github.com/DIMO-Network/edge-network/config"
 	"github.com/google/uuid"
 	"github.com/jarcoal/httpmock"
 	"github.com/rs/zerolog"
@@ -28,7 +28,7 @@ func TestCertificateService_GetOauthToken(t *testing.T) {
 	defer httpmock.DeactivateAndReset()
 	const autoPiBaseURL = "http://192.168.4.1:9000"
 	const etherAddr = "b794f5"
-	var serial uuid.UUID = uuid.New()
+	var serial = uuid.New()
 
 	mockCtrl := gomock.NewController(t)
 	defer mockCtrl.Finish()
@@ -38,18 +38,23 @@ func TestCertificateService_GetOauthToken(t *testing.T) {
 		Str("app", "edge-network").
 		Logger()
 
-	cs := NewCertificateService(logger, gateways.Development, nil, mockFileSystem())
+	// read config file
+	config, confErr := dimoConfig.ReadConfigFromPath("../config-dev.yaml")
+	if confErr != nil {
+		logger.Fatal().Err(confErr).Msg("unable to read config file")
+	}
+	cs := NewCertificateService(logger, *config, nil, mockFileSystem())
 
 	// when
 	psPath := fmt.Sprintf("/dongle/%s/execute_raw", serial.String())
 	httpmock.RegisterResponder(http.MethodPost, autoPiBaseURL+psPath,
 		httpmock.NewStringResponder(200, `{"value": "0x064493aF03c949d58EE03Df0e771B6Eb19A1018A"}`))
 
 	// Set up the expectation for the PostForm call
-	httpmock.RegisterResponder(http.MethodPost, "https://auth.dev.dimo.zone/auth/web3/generate_challenge",
+	httpmock.RegisterResponder(http.MethodPost, config.Services.Auth.Host+"/auth/web3/generate_challenge",
 		httpmock.NewStringResponder(200, `{"state": "oae7fkpeyxdatezkac5lzmo2p","challenge": "auth.dimo.zone wants you to sign in with your Ethereum account:\n0x064493aF03c949d58EE03Df0e771B6Eb19A1018A\n\n127.0.0.1 is asking you sign in.\n\nURI: https://auth.dimo.zone\nVersion: 1\nChain ID: 1\nNonce: zrIC3hmEvCsv8exZxjsMBYhEciu7oB\nIssued At: 2024-05-09T16:11:21Z"}`))
 	// set up the expectation for the Post call
-	httpmock.RegisterResponder(http.MethodPost, "https://auth.dev.dimo.zone/auth/web3/submit_challenge",
+	httpmock.RegisterResponder(http.MethodPost, config.Services.Auth.Host+"/auth/web3/submit_challenge",
 		httpmock.NewStringResponder(200, `{"access_token": "eyJhbGciOiJSUzI1NiIsImtpZCI6ImMzZWVhNzJjNDFjMzJlMjg2YThhZTc3ZmE5OTA1NmQ2YjA3ZjAxMjUifQ.eyJpc3MiOiJodHRwczovL2F1dGguZGV2LmRpbW8uem9uZSIsInByb3ZpZGVyX2lkIjoid2ViMyIsInN1YiI6IkNpb3dlRGs0UkRjNFpEY3hNVU13WldNMU5EUkdObVppTldRMU5HWmpaalkxTlRsRFJqUXhOVFEyWVRrU0JIZGxZak0iLCJhdWQiOiJzdGVwLWNhIiwiZXhwIjoxNzE2NDg5MzgyLCJpYXQiOjE3MTUyNzk3ODIsImF0X2hhc2giOiJNeDNJc3F4T2xYN0w0WVlyMVFsWFN3IiwiZW1haWxfdmVyaWZpZWQiOmZhbHNlLCJldGhlcmV1bV9hZGRyZXNzIjoiMHg5OEQ3OGQ3MTFDMGVjNTQ0RjZmYjVkNTRmY2Y2NTU5Q0Y0MTU0NmE5In0.nOgnxTtAHTX-HKaRet1yAKvIC91XehgS33MrdGUAWrdgmDWhfJykevMlnQxolDrykE8-foTDaB-ePpbr1vtcMfQ2cPhGZTJyI0nWEGNUK0qEYO4tMzgBwUGtTL6-CR3q_qTLu7DJ71_znbYxKgzVJHvsOJEju_vDKo9g2gtoaAUqUC_xN12jyhOsjn1ZVBEaXfkduoLtJgB5RdmoD8P-PGArkccBGwSKc6iCO8M2UH901WfdL8Zoh8D1-jqwaq-KdNAvyumj4viWPHys0mAXCnEqgmlfXcBaFSuNhLUck1G7Tjgs6KfYY6QkSGJapCo-RsuI5DD3jWTh396bR6o0iw"}`))
 
 	// then
@@ -78,7 +83,12 @@ func TestCertificateService_SignWeb3Certificate(t *testing.T) {
 		Str("app", "edge-network").
 		Logger()
 
-	cs := NewCertificateService(logger, gateways.Development, mockSigner, mockFileSystem())
+	// read config file
+	config, confErr := dimoConfig.ReadConfigFromPath("../config-dev.yaml")
+	if confErr != nil {
+		logger.Fatal().Err(confErr).Msg("unable to read config file")
+	}
+	cs := NewCertificateService(logger, *config, mockSigner, mockFileSystem())
 
 	// when
 	psPath := fmt.Sprintf("/dongle/%s/execute_raw", serial.String())
@@ -89,10 +99,10 @@ func TestCertificateService_SignWeb3Certificate(t *testing.T) {
 	cert := generateCert()
 	mockSigner.EXPECT().Sign(gomock.Any()).Return(&api.SignResponse{ServerPEM: api.Certificate{Certificate: cert}}, nil)
 	// Set up the expectation for the PostForm call
-	httpmock.RegisterResponder(http.MethodPost, "https://auth.dev.dimo.zone/auth/web3/generate_challenge",
+	httpmock.RegisterResponder(http.MethodPost, config.Services.Auth.Host+"/auth/web3/generate_challenge",
 		httpmock.NewStringResponder(200, `{"state": "oae7fkpeyxdatezkac5lzmo2p","challenge": "auth.dimo.zone wants you to sign in with your Ethereum account:\n0x064493aF03c949d58EE03Df0e771B6Eb19A1018A\n\n127.0.0.1 is asking you sign in.\n\nURI: https://auth.dimo.zone\nVersion: 1\nChain ID: 1\nNonce: zrIC3hmEvCsv8exZxjsMBYhEciu7oB\nIssued At: 2024-05-09T16:11:21Z"}`))
 	// set up the expectation for the Post call
-	httpmock.RegisterResponder(http.MethodPost, "https://auth.dev.dimo.zone/auth/web3/submit_challenge",
+	httpmock.RegisterResponder(http.MethodPost, config.Services.Auth.Host+"/auth/web3/submit_challenge",
 		httpmock.NewStringResponder(200, `{"access_token": "eyJhbGciOiJSUzI1NiIsImtpZCI6ImMzZWVhNzJjNDFjMzJlMjg2YThhZTc3ZmE5OTA1NmQ2YjA3ZjAxMjUifQ.eyJpc3MiOiJodHRwczovL2F1dGguZGV2LmRpbW8uem9uZSIsInByb3ZpZGVyX2lkIjoid2ViMyIsInN1YiI6IkNpb3dlRGs0UkRjNFpEY3hNVU13WldNMU5EUkdObVppTldRMU5HWmpaalkxTlRsRFJqUXhOVFEyWVRrU0JIZGxZak0iLCJhdWQiOiJzdGVwLWNhIiwiZXhwIjoxNzE2NDg5MzgyLCJpYXQiOjE3MTUyNzk3ODIsImF0X2hhc2giOiJNeDNJc3F4T2xYN0w0WVlyMVFsWFN3IiwiZW1haWxfdmVyaWZpZWQiOmZhbHNlLCJldGhlcmV1bV9hZGRyZXNzIjoiMHg5OEQ3OGQ3MTFDMGVjNTQ0RjZmYjVkNTRmY2Y2NTU5Q0Y0MTU0NmE5In0.nOgnxTtAHTX-HKaRet1yAKvIC91XehgS33MrdGUAWrdgmDWhfJykevMlnQxolDrykE8-foTDaB-ePpbr1vtcMfQ2cPhGZTJyI0nWEGNUK0qEYO4tMzgBwUGtTL6-CR3q_qTLu7DJ71_znbYxKgzVJHvsOJEju_vDKo9g2gtoaAUqUC_xN12jyhOsjn1ZVBEaXfkduoLtJgB5RdmoD8P-PGArkccBGwSKc6iCO8M2UH901WfdL8Zoh8D1-jqwaq-KdNAvyumj4viWPHys0mAXCnEqgmlfXcBaFSuNhLUck1G7Tjgs6KfYY6QkSGJapCo-RsuI5DD3jWTh396bR6o0iw"}`))
 
 	// then

config-dev.yaml

@@ -0,0 +1,27 @@
+mqtt:
+  broker:
+    host: ssl://stream.dev.dimo.zone
+    port: 8884
+    tls:
+      enabled: true
+  topics:
+    status: devices/%s/status
+    network: devices/%s/network
+    logs: devices/%s/logs
+    fingerprint: devices/%s/fingerprint
+services:
+  auth:
+    host: https://auth.dev.dimo.zone
+    clientId: step-ca
+    clientSecret: replace-me
+    caFingerprint: replace-me
+    generateChallengeURI: /auth/web3/generate_challenge
+    submitChallengeURI: /auth/web3/submit_challenge
+  ca:
+    host: https://ca.dev.dimo.zone
+    certPath: /opt/autopi/client.crt
+    privateKeyPath: /opt/autopi/client.pem
+  identity:
+    host: https://identity-api.dev.dimo.zone/query
+  vehicle:
+    host: https://vehicle-signal-decoding.dev.dimo.zone

config.yaml

@@ -0,0 +1,23 @@
+mqtt:
+  broker:
+    host: tcp://localhost
+    port: 1883
+    tls:
+      enabled: false
+  topics:
+    status: status
+    network: network
+    logs: logs
+    fingerprint: fingerprint
+services:
+  auth:
+    host: https://auth.dimo.zone
+    clientId: step-ca
+    clientSecret: replace-me
+    caFingerprint: replace-me
+  ca:
+    host: https://ca.dimo.zone
+  identity:
+    host: https://identity-api.dimo.zone/query
+  vehicle:
+    host: https://vehicle-signal-decoding.dimo.zone