Add common interface to write exported files

cmd/command_utils.go

@@ -0,0 +1,137 @@
+package cmd
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+)
+
+type CloudStorage interface {
+	UploadTo(credentialsPath, bucket, path string) error
+}
+
+func createOutputFile(filepath string) error {
+	var _, err = os.Stat(filepath)
+	if os.IsNotExist(err) {
+		var _, err = os.Create(filepath)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func mustOutFile(path string) *os.File {
+	absolutePath, err := filepath.Abs(path)
+	if err != nil {
+		cmdLogger.Fatal("could not get absolute filepath: ", err)
+	}
+
+	err = os.MkdirAll(filepath.Dir(path), os.ModePerm)
+	if err != nil {
+		cmdLogger.Fatalf("could not create directory %s: %s", path, err)
+	}
+
+	err = createOutputFile(absolutePath)
+	if err != nil {
+		cmdLogger.Fatal("could not create output file: ", err)
+	}
+
+	outFile, err := os.OpenFile(absolutePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
+	if err != nil {
+		cmdLogger.Fatal("error in opening output file: ", err)
+	}
+
+	return outFile
+}
+
+func exportEntry(entry interface{}, outFile *os.File, extra map[string]string) (int, error) {
+	// This extra marshalling/unmarshalling is silly, but it's required to properly handle the null.[String|Int*] types, and add the extra fields.
+	m, err := json.Marshal(entry)
+	if err != nil {
+		cmdLogger.Errorf("Error marshalling %+v: %v ", entry, err)
+	}
+	i := map[string]interface{}{}
+	// Use a decoder here so that 'UseNumber' ensures large ints are properly decoded
+	decoder := json.NewDecoder(bytes.NewReader(m))
+	decoder.UseNumber()
+	err = decoder.Decode(&i)
+	if err != nil {
+		cmdLogger.Errorf("Error unmarshalling %+v: %v ", i, err)
+	}
+	for k, v := range extra {
+		i[k] = v
+	}
+
+	marshalled, err := json.Marshal(i)
+	if err != nil {
+		return 0, fmt.Errorf("could not json encode %+v: %s", entry, err)
+	}
+	cmdLogger.Debugf("Writing entry to %s", outFile.Name())
+	numBytes, err := outFile.Write(marshalled)
+	if err != nil {
+		cmdLogger.Errorf("Error writing %+v to file: %s", entry, err)
+	}
+	newLineNumBytes, err := outFile.WriteString("\n")
+	if err != nil {
+		cmdLogger.Errorf("Error writing new line to file %s: %s", outFile.Name(), err)
+	}
+	return numBytes + newLineNumBytes, nil
+}
+
+// Prints the number of attempted, failed, and successful transformations as a JSON object
+func printTransformStats(attempts, failures int) {
+	resultsMap := map[string]int{
+		"attempted_transforms":  attempts,
+		"failed_transforms":     failures,
+		"successful_transforms": attempts - failures,
+	}
+
+	results, err := json.Marshal(resultsMap)
+	if err != nil {
+		cmdLogger.Fatal("Could not marshal results: ", err)
+	}
+
+	cmdLogger.Info(string(results))
+}
+
+func exportFilename(start, end uint32, dataType string) string {
+	return fmt.Sprintf("%d-%d-%s.txt", start, end-1, dataType)
+}
+
+func deleteLocalFiles(path string) {
+	err := os.RemoveAll(path)
+	if err != nil {
+		cmdLogger.Errorf("Unable to remove %s: %s", path, err)
+		return
+	}
+	cmdLogger.Infof("Successfully deleted %s", path)
+}
+
+func maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path string) {
+	if cloudProvider == "" {
+		cmdLogger.Info("No cloud provider specified for upload. Skipping upload.")
+		return
+	}
+
+	if len(cloudStorageBucket) == 0 {
+		cmdLogger.Error("No bucket specified")
+		return
+	}
+
+	var cloudStorage CloudStorage
+	switch cloudProvider {
+	case "gcp":
+		cloudStorage = newGCS(cloudCredentials, cloudStorageBucket)
+		err := cloudStorage.UploadTo(cloudCredentials, cloudStorageBucket, path)
+		if err != nil {
+			cmdLogger.Errorf("Unable to upload output to GCS: %s", err)
+			return
+		}
+	default:
+		cmdLogger.Error("Unknown cloud provider")
+	}
+}

cmd/export_account_signers.go

@@ -26,7 +26,7 @@ the export_ledger_entry_changes command.`,
 		cmdLogger.StrictExport = strictExport
 		env := utils.GetEnvironmentDetails(isTest, isFuture)
 		path := utils.MustBucketFlags(cmd.Flags(), cmdLogger)
-		gcsBucket, gcpCredentials := utils.MustGcsFlags(cmd.Flags(), cmdLogger)
+		cloudStorageBucket, cloudCredentials, cloudProvider := utils.MustCloudStorageFlags(cmd.Flags(), cmdLogger)
 
 		accounts, err := input.GetEntriesFromGenesis(endNum, xdr.LedgerEntryTypeAccount, env.ArchiveURLs)
 		if err != nil {
@@ -65,14 +65,14 @@ the export_ledger_entry_changes command.`,
 
 		printTransformStats(numSigners, numFailures)
 
-		maybeUpload(gcpCredentials, gcsBucket, path)
+		maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path)
 	},
 }
 
 func init() {
 	rootCmd.AddCommand(accountSignersCmd)
 	utils.AddCommonFlags(accountSignersCmd.Flags())
 	utils.AddBucketFlags("signers", accountSignersCmd.Flags())
-	utils.AddGcsFlags(accountSignersCmd.Flags())
+	utils.AddCloudStorageFlags(accountSignersCmd.Flags())
 	accountSignersCmd.MarkFlagRequired("end-ledger")
 }

cmd/export_accounts.go

@@ -26,7 +26,7 @@ the export_ledger_entry_changes command.`,
 		cmdLogger.StrictExport = strictExport
 		env := utils.GetEnvironmentDetails(isTest, isFuture)
 		path := utils.MustBucketFlags(cmd.Flags(), cmdLogger)
-		gcsBucket, gcpCredentials := utils.MustGcsFlags(cmd.Flags(), cmdLogger)
+		cloudStorageBucket, cloudCredentials, cloudProvider := utils.MustCloudStorageFlags(cmd.Flags(), cmdLogger)
 
 		accounts, err := input.GetEntriesFromGenesis(endNum, xdr.LedgerEntryTypeAccount, env.ArchiveURLs)
 		if err != nil {
@@ -59,15 +59,15 @@ the export_ledger_entry_changes command.`,
 
 		printTransformStats(len(accounts), numFailures)
 
-		maybeUpload(gcpCredentials, gcsBucket, path)
+		maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path)
 	},
 }
 
 func init() {
 	rootCmd.AddCommand(accountsCmd)
 	utils.AddCommonFlags(accountsCmd.Flags())
 	utils.AddBucketFlags("accounts", accountsCmd.Flags())
-	utils.AddGcsFlags(accountsCmd.Flags())
+	utils.AddCloudStorageFlags(accountsCmd.Flags())
 	accountsCmd.MarkFlagRequired("end-ledger")
 	/*
 		Current flags:

cmd/export_assets.go

@@ -19,7 +19,7 @@ var assetsCmd = &cobra.Command{
 		endNum, strictExport, isTest, isFuture, extra := utils.MustCommonFlags(cmd.Flags(), cmdLogger)
 		cmdLogger.StrictExport = strictExport
 		startNum, path, limit := utils.MustArchiveFlags(cmd.Flags(), cmdLogger)
-		gcsBucket, gcpCredentials := utils.MustGcsFlags(cmd.Flags(), cmdLogger)
+		cloudStorageBucket, cloudCredentials, cloudProvider := utils.MustCloudStorageFlags(cmd.Flags(), cmdLogger)
 
 		outFile := mustOutFile(path)
 
@@ -61,15 +61,15 @@ var assetsCmd = &cobra.Command{
 
 		printTransformStats(len(paymentOps), numFailures)
 
-		maybeUpload(gcpCredentials, gcsBucket, path)
+		maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path)
 	},
 }
 
 func init() {
 	rootCmd.AddCommand(assetsCmd)
 	utils.AddCommonFlags(assetsCmd.Flags())
 	utils.AddArchiveFlags("assets", assetsCmd.Flags())
-	utils.AddGcsFlags(assetsCmd.Flags())
+	utils.AddCloudStorageFlags(assetsCmd.Flags())
 	assetsCmd.MarkFlagRequired("end-ledger")
 
 	/*

cmd/export_claimable_balances.go

@@ -26,7 +26,7 @@ var claimableBalancesCmd = &cobra.Command{
 		cmdLogger.StrictExport = strictExport
 		env := utils.GetEnvironmentDetails(isTest, isFuture)
 		path := utils.MustBucketFlags(cmd.Flags(), cmdLogger)
-		gcsBucket, gcpCredentials := utils.MustGcsFlags(cmd.Flags(), cmdLogger)
+		cloudStorageBucket, cloudCredentials, cloudProvider := utils.MustCloudStorageFlags(cmd.Flags(), cmdLogger)
 
 		balances, err := input.GetEntriesFromGenesis(endNum, xdr.LedgerEntryTypeClaimableBalance, env.ArchiveURLs)
 		if err != nil {
@@ -59,15 +59,15 @@ var claimableBalancesCmd = &cobra.Command{
 
 		printTransformStats(len(balances), numFailures)
 
-		maybeUpload(gcpCredentials, gcsBucket, path)
+		maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path)
 	},
 }
 
 func init() {
 	rootCmd.AddCommand(claimableBalancesCmd)
 	utils.AddCommonFlags(claimableBalancesCmd.Flags())
 	utils.AddBucketFlags("claimable_balances", claimableBalancesCmd.Flags())
-	utils.AddGcsFlags(claimableBalancesCmd.Flags())
+	utils.AddCloudStorageFlags(claimableBalancesCmd.Flags())
 	claimableBalancesCmd.MarkFlagRequired("end-ledger")
 
 	/*

cmd/export_config_setting.go

@@ -23,7 +23,7 @@ var configSettingCmd = &cobra.Command{
 		cmdLogger.StrictExport = strictExport
 		env := utils.GetEnvironmentDetails(isTest, isFuture)
 		path := utils.MustBucketFlags(cmd.Flags(), cmdLogger)
-		gcsBucket, gcpCredentials := utils.MustGcsFlags(cmd.Flags(), cmdLogger)
+		cloudStorageBucket, cloudCredentials, cloudProvider := utils.MustCloudStorageFlags(cmd.Flags(), cmdLogger)
 
 		settings, err := input.GetEntriesFromGenesis(endNum, xdr.LedgerEntryTypeConfigSetting, env.ArchiveURLs)
 		if err != nil {
@@ -54,7 +54,7 @@ var configSettingCmd = &cobra.Command{
 		cmdLogger.Info("Number of bytes written: ", totalNumBytes)
 
 		printTransformStats(len(settings), numFailures)
-		maybeUpload(gcpCredentials, gcsBucket, path)
+		maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path)
 
 	},
 }
@@ -63,7 +63,7 @@ func init() {
 	rootCmd.AddCommand(configSettingCmd)
 	utils.AddCommonFlags(configSettingCmd.Flags())
 	utils.AddBucketFlags("config_settings", configSettingCmd.Flags())
-	utils.AddGcsFlags(configSettingCmd.Flags())
+	utils.AddCloudStorageFlags(configSettingCmd.Flags())
 	configSettingCmd.MarkFlagRequired("end-ledger")
 	/*
 		Current flags:

cmd/export_contract_code.go

@@ -23,7 +23,7 @@ var codeCmd = &cobra.Command{
 		cmdLogger.StrictExport = strictExport
 		env := utils.GetEnvironmentDetails(isTest, isFuture)
 		path := utils.MustBucketFlags(cmd.Flags(), cmdLogger)
-		gcsBucket, gcpCredentials := utils.MustGcsFlags(cmd.Flags(), cmdLogger)
+		cloudStorageBucket, cloudCredentials, cloudProvider := utils.MustCloudStorageFlags(cmd.Flags(), cmdLogger)
 
 		codes, err := input.GetEntriesFromGenesis(endNum, xdr.LedgerEntryTypeContractCode, env.ArchiveURLs)
 		if err != nil {
@@ -54,7 +54,7 @@ var codeCmd = &cobra.Command{
 		cmdLogger.Info("Number of bytes written: ", totalNumBytes)
 
 		printTransformStats(len(codes), numFailures)
-		maybeUpload(gcpCredentials, gcsBucket, path)
+		maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path)
 
 	},
 }
@@ -63,7 +63,7 @@ func init() {
 	rootCmd.AddCommand(codeCmd)
 	utils.AddCommonFlags(codeCmd.Flags())
 	utils.AddBucketFlags("contract_code", codeCmd.Flags())
-	utils.AddGcsFlags(codeCmd.Flags())
+	utils.AddCloudStorageFlags(codeCmd.Flags())
 	codeCmd.MarkFlagRequired("end-ledger")
 	/*
 		Current flags:

cmd/export_contract_data.go

@@ -23,7 +23,7 @@ var dataCmd = &cobra.Command{
 		cmdLogger.StrictExport = strictExport
 		env := utils.GetEnvironmentDetails(isTest, isFuture)
 		path := utils.MustBucketFlags(cmd.Flags(), cmdLogger)
-		gcsBucket, gcpCredentials := utils.MustGcsFlags(cmd.Flags(), cmdLogger)
+		cloudStorageBucket, cloudCredentials, cloudProvider := utils.MustCloudStorageFlags(cmd.Flags(), cmdLogger)
 
 		datas, err := input.GetEntriesFromGenesis(endNum, xdr.LedgerEntryTypeContractData, env.ArchiveURLs)
 		if err != nil {
@@ -59,7 +59,7 @@ var dataCmd = &cobra.Command{
 		cmdLogger.Info("Number of bytes written: ", totalNumBytes)
 
 		printTransformStats(len(datas), numFailures)
-		maybeUpload(gcpCredentials, gcsBucket, path)
+		maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path)
 
 	},
 }
@@ -68,7 +68,7 @@ func init() {
 	rootCmd.AddCommand(dataCmd)
 	utils.AddCommonFlags(dataCmd.Flags())
 	utils.AddBucketFlags("contract_data", dataCmd.Flags())
-	utils.AddGcsFlags(dataCmd.Flags())
+	utils.AddCloudStorageFlags(dataCmd.Flags())
 	dataCmd.MarkFlagRequired("end-ledger")
 	/*
 		Current flags:

cmd/export_diagnostic_events.go

@@ -19,7 +19,7 @@ var diagnosticEventsCmd = &cobra.Command{
 		endNum, strictExport, isTest, isFuture, extra := utils.MustCommonFlags(cmd.Flags(), cmdLogger)
 		cmdLogger.StrictExport = strictExport
 		startNum, path, limit := utils.MustArchiveFlags(cmd.Flags(), cmdLogger)
-		gcsBucket, gcpCredentials := utils.MustGcsFlags(cmd.Flags(), cmdLogger)
+		cloudStorageBucket, cloudCredentials, cloudProvider := utils.MustCloudStorageFlags(cmd.Flags(), cmdLogger)
 		env := utils.GetEnvironmentDetails(isTest, isFuture)
 
 		transactions, err := input.GetTransactions(startNum, endNum, limit, env)
@@ -55,15 +55,15 @@ var diagnosticEventsCmd = &cobra.Command{
 
 		printTransformStats(len(transactions), numFailures)
 
-		maybeUpload(gcpCredentials, gcsBucket, path)
+		maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path)
 	},
 }
 
 func init() {
 	rootCmd.AddCommand(diagnosticEventsCmd)
 	utils.AddCommonFlags(diagnosticEventsCmd.Flags())
 	utils.AddArchiveFlags("diagnostic_events", diagnosticEventsCmd.Flags())
-	utils.AddGcsFlags(diagnosticEventsCmd.Flags())
+	utils.AddCloudStorageFlags(diagnosticEventsCmd.Flags())
 	diagnosticEventsCmd.MarkFlagRequired("end-ledger")
 
 	/*

cmd/export_effects.go

@@ -16,7 +16,7 @@ var effectsCmd = &cobra.Command{
 		endNum, strictExport, isTest, isFuture, extra := utils.MustCommonFlags(cmd.Flags(), cmdLogger)
 		cmdLogger.StrictExport = strictExport
 		startNum, path, limit := utils.MustArchiveFlags(cmd.Flags(), cmdLogger)
-		gcsBucket, gcpCredentials := utils.MustGcsFlags(cmd.Flags(), cmdLogger)
+		cloudStorageBucket, cloudCredentials, cloudProvider := utils.MustCloudStorageFlags(cmd.Flags(), cmdLogger)
 		env := utils.GetEnvironmentDetails(isTest, isFuture)
 
 		transactions, err := input.GetTransactions(startNum, endNum, limit, env)
@@ -53,15 +53,15 @@ var effectsCmd = &cobra.Command{
 
 		printTransformStats(len(transactions), numFailures)
 
-		maybeUpload(gcpCredentials, gcsBucket, path)
+		maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path)
 	},
 }
 
 func init() {
 	rootCmd.AddCommand(effectsCmd)
 	utils.AddCommonFlags(effectsCmd.Flags())
 	utils.AddArchiveFlags("effects", effectsCmd.Flags())
-	utils.AddGcsFlags(effectsCmd.Flags())
+	utils.AddCloudStorageFlags(effectsCmd.Flags())
 	effectsCmd.MarkFlagRequired("end-ledger")
 
 	/*