Merge p23 into master

.github/workflows/galexie-release.yml

@@ -13,12 +13,12 @@ jobs:
       GALEXIE_INTEGRATION_TESTS_ENABLED: "true"
       GALEXIE_INTEGRATION_TESTS_CAPTIVE_CORE_BIN: /usr/bin/stellar-core
       # this pins to a version of quickstart:testing that has the same version as STELLAR_CORE_VERSION
-      # this is the multi-arch index sha, get it by 'docker buildx imagetools inspect stellar/quickstart:testing'
-      GALEXIE_INTEGRATION_TESTS_QUICKSTART_IMAGE: docker.io/stellar/quickstart:testing@sha256:5333ec87069efd7bb61f6654a801dc093bf0aad91f43a5ba84806d3efe4a6322
+      # this is the multi-arch index sha, get it by 'docker buildx imagetools inspect stellar/quickstart:future'
+      GALEXIE_INTEGRATION_TESTS_QUICKSTART_IMAGE: docker.io/stellar/quickstart:future@sha256:ea7f4dd4c8e1dc4eb69194ef5b9659aa73e08a89146ea80acfc2fdc073fffb32
       GALEXIE_INTEGRATION_TESTS_QUICKSTART_IMAGE_PULL: "false"
       GALEXIE_INTEGRATION_TESTS_LOCALSTACK_IMAGE_TAG: "4.6.0"
       GALEXIE_INTEGRATION_TESTS_LOCALSTACK_IMAGE_PULL: "false"
-      STELLAR_CORE_VERSION: 22.3.0-2485.e643061a4.focal
+      STELLAR_CORE_VERSION: 23.0.0-2634.d5cbc0793.focal
     steps:
       - name: Set VERSION
         run: |

.github/workflows/galexie.yml

@@ -13,12 +13,12 @@ jobs:
       matrix:
         storage_type: [ GCS, S3 ]
     env:
-      CAPTIVE_CORE_DEBIAN_PKG_VERSION: 22.3.0-2485.e643061a4.focal
+      CAPTIVE_CORE_DEBIAN_PKG_VERSION: 23.0.0-2634.d5cbc0793.focal
       GALEXIE_INTEGRATION_TESTS_ENABLED: "true"
       GALEXIE_INTEGRATION_TESTS_CAPTIVE_CORE_BIN: /usr/bin/stellar-core
       # this pins to a version of quickstart:testing that has the same version as GALEXIE_INTEGRATION_TESTS_CAPTIVE_CORE_BIN
-      # this is the multi-arch index sha, get it by 'docker buildx imagetools inspect stellar/quickstart:testing'
-      GALEXIE_INTEGRATION_TESTS_QUICKSTART_IMAGE: docker.io/stellar/quickstart:testing@sha256:5333ec87069efd7bb61f6654a801dc093bf0aad91f43a5ba84806d3efe4a6322
+      # this is the multi-arch index sha, get it by 'docker buildx imagetools inspect stellar/quickstart:future'
+      GALEXIE_INTEGRATION_TESTS_QUICKSTART_IMAGE: docker.io/stellar/quickstart:future@sha256:ea7f4dd4c8e1dc4eb69194ef5b9659aa73e08a89146ea80acfc2fdc073fffb32
       GALEXIE_INTEGRATION_TESTS_QUICKSTART_IMAGE_PULL: "false"
       GALEXIE_INTEGRATION_TESTS_LOCALSTACK_IMAGE_TAG: "4.6.0"
       GALEXIE_INTEGRATION_TESTS_LOCALSTACK_IMAGE_PULL: "false"

.github/workflows/golangci-lint.yml

@@ -26,7 +26,4 @@ jobs:
           args: -v --issues-exit-code=0 --config=.golangci.yml # exit without errors for now - won't fail the build
           github-token: ${{ secrets.GITHUB_TOKEN }}
           only-new-issues: true
-
-
-
-
+        continue-on-error: true

.github/workflows/horizon.yml

@@ -12,8 +12,8 @@ jobs:
       matrix:
         os: [ubuntu-22.04]
         go: ["1.22", "1.23"]
-        pg: [12, 16]
-        protocol-version: [22]
+        pg: [14, 16]
+        protocol-version: [22,23]
     runs-on: ${{ matrix.os }}
     services:
       postgres:
@@ -32,10 +32,12 @@ jobs:
     env:
       HORIZON_INTEGRATION_TESTS_ENABLED: true
       HORIZON_INTEGRATION_TESTS_CORE_MAX_SUPPORTED_PROTOCOL: ${{ matrix.protocol-version }}
-      HORIZON_INTEGRATION_TESTS_CAPTIVE_CORE_USE_DB: true
-      PROTOCOL_22_CORE_DEBIAN_PKG_VERSION: 22.3.0-2485.e643061a4.focal
-      PROTOCOL_22_CORE_DOCKER_IMG: stellar/stellar-core:22.3.0-2485.e643061a4.focal
-      PROTOCOL_22_STELLAR_RPC_DOCKER_IMG: stellar/stellar-rpc:22.1.2
+      PROTOCOL_22_CORE_DEBIAN_PKG_VERSION: 23.0.0-2634.d5cbc0793.focal
+      PROTOCOL_22_CORE_DOCKER_IMG: stellar/unsafe-stellar-core:23.0.0-2634.d5cbc0793.focal
+      PROTOCOL_22_STELLAR_RPC_DOCKER_IMG: stellar/stellar-rpc:23.0.0-rc2-126
+      PROTOCOL_23_CORE_DEBIAN_PKG_VERSION: 23.0.0-2634.d5cbc0793.focal
+      PROTOCOL_23_CORE_DOCKER_IMG: stellar/unsafe-stellar-core:23.0.0-2634.d5cbc0793.focal
+      PROTOCOL_23_STELLAR_RPC_DOCKER_IMG: stellar/stellar-rpc:23.0.0-rc2-126
       PGHOST: localhost
       PGPORT: 5432
       PGUSER: postgres
@@ -79,8 +81,8 @@ jobs:
       - name: Install core
         run: |
           # Workaround for https://github.com/actions/virtual-environments/issues/5245,
-          # libc++1-8 won't be installed if another version is installed (but apt won't give you a helpul
-          # message about why the installation fails)
+          # libc++1-8 won't be installed if another version is installed (but
+          # apt won't give you a helpul message about why the installation fails)
           sudo apt list --installed | grep libc++
           sudo apt-get remove -y libc++1-* libc++abi1-* || true
 
@@ -98,7 +100,7 @@ jobs:
       - name: Calculate the source hash
         id: calculate_source_hash
         run: |
-          combined_hash=$(echo "horizon-hash-${{ hashFiles('./horizon') }}-${{ hashFiles('./clients/horizonclient/**') }}-${{ hashFiles('./protocols/horizon/**') }}-${{ hashFiles('./txnbuild/**') }}-${{ hashFiles('./ingest/**') }}-${{ hashFiles('./xdr/**') }}-${{ hashFiles('./services/**') }}-${{ env.PROTOCOL_21_CORE_DOCKER_IMG }}-${{ env.PROTOCOL_21_RPC_DOCKER_IMG }}-${{ env.PROTOCOL_21_CORE_DEBIAN_PKG_VERSION }}-${{ env.PREFIX }}" | sha256sum | cut -d ' ' -f 1)
+          combined_hash=$(echo "horizon-hash-${{ hashFiles('./horizon') }}-${{ hashFiles('./clients/horizonclient/**') }}-${{ hashFiles('./protocols/horizon/**') }}-${{ hashFiles('./txnbuild/**') }}-${{ hashFiles('./ingest/**') }}-${{ hashFiles('./xdr/**') }}-${{ hashFiles('./services/**') }}-${{ env.PROTOCOL_23_CORE_DOCKER_IMG }}-${{ env.PROTOCOL_23_RPC_DOCKER_IMG }}-${{ env.PROTOCOL_23_CORE_DEBIAN_PKG_VERSION }}-${{ env.PREFIX }}" | sha256sum | cut -d ' ' -f 1)
           echo "COMBINED_SOURCE_HASH=$combined_hash" >> "$GITHUB_ENV"
 
       - name: Restore Horizon binary and integration tests source hash to cache
@@ -124,7 +126,7 @@ jobs:
     runs-on: ubuntu-22.04
     env:
       GO_VERSION: 1.23.4
-      STELLAR_CORE_VERSION: 22.3.0-2485.e643061a4.jammy
+      STELLAR_CORE_VERSION: 23.0.0-2634.d5cbc0793.jammy
     steps:
       - uses: actions/checkout@v3
         with:
@@ -135,7 +137,11 @@ jobs:
         run: |
           docker build --build-arg="GO_VERSION=$GO_VERSION" --build-arg="STELLAR_CORE_VERSION=$STELLAR_CORE_VERSION" -f services/horizon/docker/verify-range/Dockerfile -t stellar/horizon-verify-range services/horizon/docker/verify-range/
           # Any range should do for basic testing, this range was chosen pretty early in history so that it only takes a few mins to run
-          docker run -e BRANCH=$(git rev-parse HEAD) -e FROM=10000063 -e TO=10000127 stellar/horizon-verify-range
+          # TODO - remove the BASE_BRANCH override once protocol-23 merges to master,
+          # this is fine while protocol-23 branch is active.
+          # reason - master can't be used becaue the p23 version of core won't accept the same toml config params.
+          # this is still effective at least to confirm verify-range image works. it just doesn't do the compare from master.
+          docker run -e BASE_BRANCH=$(git rev-parse HEAD) -e BRANCH=$(git rev-parse HEAD) -e FROM=10000063 -e TO=10000127 stellar/horizon-verify-range
 
       # Push image
       - if: github.ref == 'refs/heads/master'

Makefile

@@ -1,7 +1,7 @@
 # Docker build targets use an optional "TAG" environment
 # variable can be set to use custom tag name. For example:
 #   TAG=my-registry.example.com/keystore:dev make keystore
-DOWNLOADABLE_XDRS = xdr/Stellar-SCP.x \
+XDRS = xdr/Stellar-SCP.x \
 xdr/Stellar-ledger-entries.x \
 xdr/Stellar-ledger.x \
 xdr/Stellar-overlay.x \
@@ -12,13 +12,12 @@ xdr/Stellar-contract-meta.x \
 xdr/Stellar-contract-spec.x \
 xdr/Stellar-contract.x \
 xdr/Stellar-internal.x \
-xdr/Stellar-contract-config-setting.x
+xdr/Stellar-contract-config-setting.x \
+xdr/Stellar-exporter.x
 
-XDRS = $(DOWNLOADABLE_XDRS) xdr/Stellar-exporter.x
 
-
-XDRGEN_COMMIT=e2cac557162d99b12ae73b846cf3d5bfe16636de
-XDR_COMMIT=529d5176f24c73eeccfa5eba481d4e89c19b1181
+XDRGEN_COMMIT=af107f07237a15fcf5f9aea71b2bebfcc1113b45
+XDR_COMMIT=4b7a2ef7931ab2ca2499be68d849f38190b443ca
 
 .PHONY: xdr xdr-clean xdr-update
 
@@ -46,15 +45,15 @@ recoverysigner:
 regulated-assets-approval-server:
 	$(MAKE) -C services/regulated-assets-approval-server/ docker-build
 
-gxdr/xdr_generated.go: $(DOWNLOADABLE_XDRS)
+gxdr/xdr_generated.go: $(XDRS)
 	go run github.com/xdrpp/goxdr/cmd/goxdr -p gxdr -enum-comments -o $@ $(XDRS)
 	gofmt -s -w $@
 
 xdr/%.x:
 	printf "%s" ${XDR_COMMIT} > xdr/xdr_commit_generated.txt
 	curl -Lsf -o $@ https://raw.githubusercontent.com/stellar/stellar-xdr/$(XDR_COMMIT)/$(@F)
 
-xdr/xdr_generated.go: $(DOWNLOADABLE_XDRS)
+xdr/xdr_generated.go: $(XDRS)
 	docker run -it --rm -v $$PWD:/wd -w /wd ruby /bin/bash -c '\
 		gem install specific_install -v 0.3.8 && \
 		gem specific_install https://github.com/stellar/xdrgen.git -b $(XDRGEN_COMMIT) && \
@@ -70,7 +69,7 @@ xdr/xdr_generated.go: $(DOWNLOADABLE_XDRS)
 xdr: gxdr/xdr_generated.go xdr/xdr_generated.go
 
 xdr-clean:
-	rm $(DOWNLOADABLE_XDRS) || true
+	rm $(XDRS) || true
 
 xdr-update: xdr-clean xdr
 

clients/stellarcore/client.go

@@ -86,8 +86,7 @@ type GenSorobanConfig struct {
 	BaseSeqNum        uint32
 	NetworkPassphrase string
 	SigningKey        *keypair.Full
-	// looks for `stellar-core` in the system PATH if empty
-	StellarCorePath string
+	StellarCoreImage  string
 }
 
 func GenSorobanConfigUpgradeTxAndKey(
@@ -96,11 +95,18 @@ func GenSorobanConfigUpgradeTxAndKey(
 	if err != nil {
 		return nil, xdr.ConfigUpgradeSetKey{}, err
 	}
-	corePath := config.StellarCorePath
-	if corePath == "" {
-		corePath = "stellar-core"
+
+	cmd := exec.Command("docker", "pull", config.StellarCoreImage)
+	_, err = cmd.Output()
+	if err != nil {
+		return nil, xdr.ConfigUpgradeSetKey{}, err
 	}
-	cmd := exec.Command(corePath, "get-settings-upgrade-txs",
+
+	cmd = exec.Command("docker",
+		"run",
+		"-i",
+		config.StellarCoreImage,
+		"get-settings-upgrade-txs",
 		config.SigningKey.Address(),
 		strconv.FormatUint(uint64(config.BaseSeqNum), 10),
 		config.NetworkPassphrase,

clients/stellarcore/client_test.go

@@ -2,12 +2,11 @@ package stellarcore
 
 import (
 	"context"
+	"crypto/rand"
 	"fmt"
 	"io"
 	"net/http"
 	"net/url"
-	"os"
-	"os/exec"
 	"testing"
 
 	"github.com/jarcoal/httpmock"
@@ -143,24 +142,54 @@ func TestGetLedgerEntries(t *testing.T) {
 	hmock := httptest.NewClient()
 	c := &Client{HTTP: hmock, URL: "http://localhost:11626"}
 
+	var hash xdr.ContractId
+	_, err := rand.Read(hash[:])
+	require.NoError(t, err)
+
+	tr, err := xdr.NewScVal(xdr.ScValTypeScvBool, true)
+	require.NoError(t, err)
+	fl, err := xdr.NewScVal(xdr.ScValTypeScvBool, false)
+	require.NoError(t, err)
+
+	rawEntry := xdr.ContractDataEntry{
+		Ext:        xdr.ExtensionPoint{},
+		Contract:   xdr.ScAddress{Type: xdr.ScAddressTypeScAddressTypeContract, ContractId: &hash},
+		Key:        tr,
+		Durability: xdr.ContractDataDurabilityPersistent,
+		Val:        fl,
+	}
+	entry := xdr.LedgerEntry{
+		LastModifiedLedgerSeq: 1210,
+		Data: xdr.LedgerEntryData{
+			Type:         xdr.LedgerEntryTypeContractData,
+			ContractData: &rawEntry,
+		},
+	}
+	entryB64, err := xdr.MarshalBase64(entry)
+	require.NoError(t, err)
+
+	var key xdr.LedgerKey
+	acc, err := xdr.AddressToAccountId(keypair.MustRandom().Address())
+	require.NoError(t, err)
+	key.SetAccount(acc)
+	keyB64, err := key.MarshalBinaryBase64()
+	require.NoError(t, err)
+
 	// build a fake response body
 	mockResp := proto.GetLedgerEntryResponse{
 		Ledger: 1215, // checkpoint align on expected request
 		Entries: []proto.LedgerEntryResponse{{
-			Entry: "pretend this is XDR lol",
-			State: "live",
-			Ttl:   1234,
+			Entry:              entryB64,
+			State:              "live",
+			LiveUntilLedgerSeq: 1234,
 		}, {
-			Entry: "pretend this is another XDR lol",
+			Entry: entryB64,
 			State: "archived",
+		}, {
+			State: "new",
 		}},
 	}
 
-	var key xdr.LedgerKey
-	acc, err := xdr.AddressToAccountId(keypair.MustRandom().Address())
-	require.NoError(t, err)
-	key.SetAccount(acc)
-
 	// happy path - fetch an entry
 	ce := hmock.On("POST", "http://localhost:11626/getledgerentry")
 	hmock.RegisterResponder(
@@ -171,9 +200,11 @@ func TestGetLedgerEntries(t *testing.T) {
 			requestData, ierr := io.ReadAll(r.Body)
 			require.NoError(t, ierr)
 
-			keyB64, ierr := key.MarshalBinaryBase64()
-			require.NoError(t, ierr)
-			expected := fmt.Sprintf("key=%s&ledgerSeq=1234", url.QueryEscape(keyB64))
+			expected := fmt.Sprintf(
+				"key=%s&key=%s&key=%s&ledgerSeq=1234",
+				url.QueryEscape(keyB64),
+				url.QueryEscape(keyB64),
+				url.QueryEscape(keyB64))
 			require.Equal(t, expected, string(requestData))
 
 			resp, ierr := httpmock.NewJsonResponse(http.StatusOK, &mockResp)
@@ -182,39 +213,36 @@ func TestGetLedgerEntries(t *testing.T) {
 			return resp, nil
 		})
 
-	resp, err := c.GetLedgerEntries(context.Background(), 1234, key)
+	resp, err := c.GetLedgerEntries(context.Background(), 1234, key, key, key)
 	require.NoError(t, err)
 	require.NotNil(t, resp)
 
 	require.EqualValues(t, 1215, resp.Ledger)
-	require.Len(t, resp.Entries, 2)
-	require.Equal(t, "pretend this is XDR lol", resp.Entries[0].Entry)
-	require.Equal(t, "pretend this is another XDR lol", resp.Entries[1].Entry)
-	require.EqualValues(t, 1234, resp.Entries[0].Ttl)
+	require.Len(t, resp.Entries, 3)
+	require.Equal(t, entryB64, resp.Entries[0].Entry)
+	require.Equal(t, entryB64, resp.Entries[1].Entry)
+	require.Empty(t, resp.Entries[2].Entry)
+	require.EqualValues(t, 1234, resp.Entries[0].LiveUntilLedgerSeq)
+	require.EqualValues(t, 0, resp.Entries[1].LiveUntilLedgerSeq)
+	require.EqualValues(t, 0, resp.Entries[2].LiveUntilLedgerSeq)
 	require.EqualValues(t, "live", resp.Entries[0].State)
 	require.EqualValues(t, "archived", resp.Entries[1].State)
+	require.EqualValues(t, "new", resp.Entries[2].State)
 
+	// TTL keys aren't returned
 	key.Type = xdr.LedgerEntryTypeTtl
 	_, err = c.GetLedgerEntries(context.Background(), 1234, key)
 	require.Error(t, err)
 }
 
 func TestGenSorobanConfigUpgradeTxAndKey(t *testing.T) {
-	coreBinary := os.Getenv("STELLAR_CORE_BINARY_PATH")
-	if coreBinary == "" {
-		var err error
-		coreBinary, err = exec.LookPath("stellar-core")
-		if err != nil {
-			t.Skip("couldn't find stellar core binary")
-		}
-	}
 	key, err := keypair.ParseFull("SB6VZS57IY25334Y6F6SPGFUNESWS7D2OSJHKDPIZ354BK3FN5GBTS6V")
 	require.NoError(t, err)
 	funcConfig := GenSorobanConfig{
 		BaseSeqNum:        1,
 		NetworkPassphrase: network.TestNetworkPassphrase,
 		SigningKey:        key,
-		StellarCorePath:   coreBinary,
+		StellarCoreImage:  "stellar/stellar-core:22",
 	}
 	config := xdr.ConfigUpgradeSet{
 		UpdatedEntry: []xdr.ConfigSettingEntry{

gxdr/dump.go

@@ -12,7 +12,13 @@ func Dump(v goxdr.XdrType) []byte {
 	var buf bytes.Buffer
 	writer := goxdr.XdrOut{Out: &buf}
 	writer.Marshal("", v)
-	return buf.Bytes()
+	output := buf.Bytes()
+	buf.Reset()
+	return output
+}
+
+func Parse(val goxdr.XdrType, in []byte) {
+	val.XdrMarshal(&goxdr.XdrIn{bytes.NewReader(in)}, "")
 }
 
 // Convert serializes the given goxdr value into another destination value