Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Updates for 20.0.0 release #209

Merged
merged 2 commits into from
Dec 8, 2023
Merged

Updates for 20.0.0 release #209

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
69b8325
Updates for 20.0.0 release
chowbao Dec 7, 2023
4b085ef
Fix typo
chowbao Dec 8, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions cmd/export_ledger_entry_changes.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ be exported.`,
"contract_data": {},
"contract_code": {},
"config_settings": {},
"expiration": {},
"ttl": {},
}

for entryType, changes := range batch.Changes {
Expand Down Expand Up @@ -236,18 +236,18 @@ be exported.`,
}
transformedOutputs["config_settings"] = append(transformedOutputs["config_settings"], configSettings)
}
case xdr.LedgerEntryTypeExpiration:
if !exports["export-expiration"] {
case xdr.LedgerEntryTypeTtl:
if !exports["export-ttl"] {
continue
}
for i, change := range changes.Changes {
expiration, err := transform.TransformExpiration(change, changes.LedgerHeaders[i])
ttl, err := transform.TransformTtl(change, changes.LedgerHeaders[i])
if err != nil {
entry, _, _, _ := utils.ExtractEntryFromChange(change)
cmdLogger.LogError(fmt.Errorf("error transforming expiration entry last updated at %d: %s", entry.LastModifiedLedgerSeq, err))
cmdLogger.LogError(fmt.Errorf("error transforming ttl entry last updated at %d: %s", entry.LastModifiedLedgerSeq, err))
continue
}
transformedOutputs["expiration"] = append(transformedOutputs["expiration"], expiration)
transformedOutputs["ttl"] = append(transformedOutputs["ttl"], ttl)
}
}
}
Expand Down
28 changes: 14 additions & 14 deletions cmd/export_expiration.go → cmd/export_ttl.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,9 @@ import (
"github.com/stellar/go/xdr"
)

var expirationCmd = &cobra.Command{
Use: "export_expiration",
Short: "Exports the expiration information.",
var ttlCmd = &cobra.Command{
Use: "export_ttl",
Short: "Exports the ttl information.",
Long: ``,
Run: func(cmd *cobra.Command, args []string) {
cmdLogger.SetLevel(logrus.InfoLevel)
Expand All @@ -25,7 +25,7 @@ var expirationCmd = &cobra.Command{
path := utils.MustBucketFlags(cmd.Flags(), cmdLogger)
gcsBucket, gcpCredentials := utils.MustGcsFlags(cmd.Flags(), cmdLogger)

expirations, err := input.GetEntriesFromGenesis(endNum, xdr.LedgerEntryTypeExpiration, env.ArchiveURLs)
ttls, err := input.GetEntriesFromGenesis(endNum, xdr.LedgerEntryTypeTtl, env.ArchiveURLs)
if err != nil {
cmdLogger.Fatal("Error getting ledger entries: ", err)
}
Expand All @@ -34,17 +34,17 @@ var expirationCmd = &cobra.Command{
numFailures := 0
totalNumBytes := 0
var header xdr.LedgerHeaderHistoryEntry
for _, expiration := range expirations {
transformed, err := transform.TransformExpiration(expiration, header)
for _, ttl := range ttls {
transformed, err := transform.TransformTtl(ttl, header)
if err != nil {
cmdLogger.LogError(fmt.Errorf("could not transform expiration %+v: %v", expiration, err))
cmdLogger.LogError(fmt.Errorf("could not transform ttl %+v: %v", ttl, err))
numFailures += 1
continue
}

numBytes, err := exportEntry(transformed, outFile, extra)
if err != nil {
cmdLogger.LogError(fmt.Errorf("could not export expiration %+v: %v", expiration, err))
cmdLogger.LogError(fmt.Errorf("could not export ttl %+v: %v", ttl, err))
numFailures += 1
continue
}
Expand All @@ -53,18 +53,18 @@ var expirationCmd = &cobra.Command{
outFile.Close()
cmdLogger.Info("Number of bytes written: ", totalNumBytes)

printTransformStats(len(expirations), numFailures)
printTransformStats(len(ttls), numFailures)
maybeUpload(gcpCredentials, gcsBucket, path)

},
}

func init() {
rootCmd.AddCommand(expirationCmd)
utils.AddCommonFlags(expirationCmd.Flags())
utils.AddBucketFlags("expiration", expirationCmd.Flags())
utils.AddGcsFlags(expirationCmd.Flags())
expirationCmd.MarkFlagRequired("end-ledger")
rootCmd.AddCommand(ttlCmd)
utils.AddCommonFlags(ttlCmd.Flags())
utils.AddBucketFlags("ttl", ttlCmd.Flags())
utils.AddGcsFlags(ttlCmd.Flags())
ttlCmd.MarkFlagRequired("end-ledger")
/*
Current flags:
end-ledger: the ledger sequence number for the end of the export range (required)
Expand Down
8 changes: 4 additions & 4 deletions cmd/export_expiration_test.go → cmd/export_ttl_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,19 +4,19 @@ import (
"testing"
)

func TestExportExpiration(t *testing.T) {
func TestExportttl(t *testing.T) {
t.Skip("Skipping due to unstable data in Futurenet")
// TODO: find ledger with data and create testdata
tests := []cliTest{
{
name: "expiration",
args: []string{"export_expiration", "-e", "78975", "-o", gotTestDir(t, "bucket_read.txt")},
name: "ttl",
args: []string{"export_ttl", "-e", "78975", "-o", gotTestDir(t, "bucket_read.txt")},
golden: "bucket_read.golden",
wantErr: nil,
},
}

for _, test := range tests {
runCLITest(t, test, "testdata/expiration/")
runCLITest(t, test, "testdata/ttl/")
}
}
2 changes: 1 addition & 1 deletion docker/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ COPY . .
RUN go build -v -o /usr/local/bin ./...

# stage 2: runtime enviroment
FROM stellar/stellar-core:20.0.0-1504.rc2.22088c1f2.focal
FROM chowbao/stellar-core:20.0.0-1615.617729910.focal
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ah, still no public docker image?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There is stellar/unsafe-stellar-core:20.0.0-1615.617729910.focal that horizon is using but the unsafe-stellar-core part is new. Not sure if they are going to release a normal stellar-core


WORKDIR /etl

Expand Down
101 changes: 49 additions & 52 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,86 +5,83 @@ go 1.19
require (
cloud.google.com/go/storage v1.32.0
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13
github.com/guregu/null v2.1.3-0.20151024101046-79c5bd36b615+incompatible
github.com/lib/pq v1.9.0
github.com/guregu/null v4.0.0+incompatible
github.com/lib/pq v1.10.9
github.com/mitchellh/go-homedir v1.1.0
github.com/pkg/errors v0.9.1
github.com/sirupsen/logrus v1.7.0
github.com/spf13/cobra v1.1.1
github.com/sirupsen/logrus v1.9.3
github.com/spf13/cobra v1.7.0
github.com/spf13/pflag v1.0.5
github.com/spf13/viper v1.7.1
github.com/stellar/go v0.0.0-20230918135710-d41faf8cd619
github.com/stretchr/testify v1.8.1
github.com/spf13/viper v1.17.0
github.com/stellar/go v0.0.0-20231206190710-208a73f6165f
github.com/stretchr/testify v1.8.4
)

require (
cloud.google.com/go v0.110.6 // indirect
cloud.google.com/go v0.110.7 // indirect
cloud.google.com/go/compute v1.23.0 // indirect
cloud.google.com/go/compute/metadata v0.2.3 // indirect
cloud.google.com/go/iam v1.1.1 // indirect
github.com/Masterminds/squirrel v1.5.0 // indirect
github.com/Microsoft/go-winio v0.4.16 // indirect
github.com/aws/aws-sdk-go v1.44.326 // indirect
github.com/Masterminds/squirrel v1.5.4 // indirect
github.com/Microsoft/go-winio v0.6.1 // indirect
github.com/aws/aws-sdk-go v1.45.26 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/go-chi/chi v4.1.2+incompatible // indirect
github.com/go-errors/errors v1.1.1 // indirect
github.com/go-sql-driver/mysql v1.5.0 // indirect
github.com/go-errors/errors v1.5.1 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/google/go-cmp v0.5.9 // indirect
github.com/google/s2a-go v0.1.5 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect
github.com/google/s2a-go v0.1.7 // indirect
github.com/google/uuid v1.3.1 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.1 // indirect
github.com/googleapis/gax-go/v2 v2.12.0 // indirect
github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 // indirect
github.com/gorilla/schema v1.2.0 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/holiman/uint256 v1.2.0 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/holiman/uint256 v1.2.3 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/jmoiron/sqlx v1.2.0 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/jmoiron/sqlx v1.3.5 // indirect
github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect
github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
github.com/magiconair/properties v1.8.4 // indirect
github.com/mattn/go-sqlite3 v1.14.5 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
github.com/mitchellh/mapstructure v1.4.0 // indirect
github.com/pelletier/go-toml v1.9.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_golang v1.11.1 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.26.0 // indirect
github.com/prometheus/procfs v0.6.0 // indirect
github.com/magiconair/properties v1.8.7 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/pelletier/go-toml v1.9.5 // indirect
github.com/pelletier/go-toml/v2 v2.1.0 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/prometheus/client_golang v1.17.0 // indirect
github.com/prometheus/client_model v0.5.0 // indirect
github.com/prometheus/common v0.44.0 // indirect
github.com/prometheus/procfs v0.12.0 // indirect
github.com/sagikazarmark/locafero v0.3.0 // indirect
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
github.com/segmentio/go-loggly v0.5.1-0.20171222203950-eb91657e62b2 // indirect
github.com/smartystreets/assertions v1.2.0 // indirect
github.com/spf13/afero v1.5.1 // indirect
github.com/spf13/cast v1.3.1 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/stellar/go-xdr v0.0.0-20211103144802-8017fc4bdfee // indirect
github.com/stretchr/objx v0.5.0 // indirect
github.com/subosito/gotenv v1.2.0 // indirect
github.com/sourcegraph/conc v0.3.0 // indirect
github.com/spf13/afero v1.10.0 // indirect
github.com/spf13/cast v1.5.1 // indirect
github.com/stellar/go-xdr v0.0.0-20231122183749-b53fb00bcac2 // indirect
github.com/stretchr/objx v0.5.1 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
go.opencensus.io v0.24.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/crypto v0.14.0 // indirect
golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 // indirect
golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
golang.org/x/mod v0.13.0 // indirect
golang.org/x/net v0.17.0 // indirect
golang.org/x/oauth2 v0.11.0 // indirect
golang.org/x/sync v0.3.0 // indirect
golang.org/x/oauth2 v0.12.0 // indirect
golang.org/x/sync v0.4.0 // indirect
golang.org/x/sys v0.13.0 // indirect
golang.org/x/text v0.13.0 // indirect
golang.org/x/tools v0.14.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/api v0.138.0 // indirect
google.golang.org/api v0.143.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230807174057-1744710a1577 // indirect
google.golang.org/grpc v1.57.1 // indirect
google.golang.org/genproto v0.0.0-20230913181813-007df8e322eb // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20230913181813-007df8e322eb // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13 // indirect
google.golang.org/grpc v1.58.3 // indirect
google.golang.org/protobuf v1.31.0 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
gopkg.in/ini.v1 v1.62.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
Loading
Loading