From a54f59e9d2e1050da9424472fef5518b463b5285 Mon Sep 17 00:00:00 2001 From: Marcelo Salhab Brogliato Date: Thu, 19 Mar 2026 13:56:27 -0500 Subject: [PATCH 1/3] feat: add support for mining transactions with shielded outputs Use local hathorlib (from ../hathor-core/hathorlib) which includes ShieldedOutputsHeader, AmountShieldedOutput, and FullShieldedOutput types. Update Dockerfile to copy hathorlib source into the build context (build from parent directory). Add 21 tests covering serialization round-trips, tx parsing, PoW solving, dev-miner HTTP API lifecycle, and TxJob handling for shielded transactions. Co-Authored-By: Claude Opus 4.6 (1M context) --- Dockerfile | 17 +- poetry.lock | 32 +- pyproject.toml | 2 +- tests/test_shielded_outputs.py | 526 +++++++++++++++++++++++++++++++++ 4 files changed, 558 insertions(+), 19 deletions(-) create mode 100644 tests/test_shielded_outputs.py diff --git a/Dockerfile b/Dockerfile index b287d82..017be87 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,16 @@ RUN apk add --no-cache gcc musl-dev libffi-dev openssl-dev rust cargo RUN pip --no-input --no-cache-dir install --upgrade pip wheel RUN pip --no-input --no-cache-dir install 'poetry>=1.2.0b2' -COPY poetry.lock pyproject.toml /code/ +# Copy hathorlib source (local path dependency: ../hathor-core/hathorlib). +# Build from the parent directory: +# docker build -f tx-mining-service/Dockerfile -t tx-mining-service . +COPY hathor-core/hathorlib/ /code/hathor-core/hathorlib/ + +# Install in a subdirectory so the relative path "../hathor-core/hathorlib" +# in pyproject.toml resolves correctly to /code/hathor-core/hathorlib/. +COPY tx-mining-service/poetry.lock tx-mining-service/pyproject.toml /code/tx-mining-service/ + +WORKDIR /code/tx-mining-service RUN poetry config virtualenvs.create false \ && poetry install --only main --no-interaction --no-ansi @@ -16,9 +25,11 @@ RUN poetry config virtualenvs.create false \ FROM python:3.11-alpine COPY --from=build /usr/local/lib/python3.11/site-packages /usr/local/lib/python3.11/site-packages +# hathorlib is installed in develop mode (.pth file points to this path) +COPY --from=build /code/hathor-core/hathorlib /code/hathor-core/hathorlib RUN apk add libgcc -COPY txstratum/ ./txstratum -COPY main.py log.conf ./ +COPY tx-mining-service/txstratum/ ./txstratum +COPY tx-mining-service/main.py tx-mining-service/log.conf ./ ENTRYPOINT ["python", "-m", "main"] diff --git a/poetry.lock b/poetry.lock index 12ef029..b0c0e44 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.3.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.3.2 and should not be changed by hand. [[package]] name = "aiohttp" @@ -616,28 +616,30 @@ files = [ [[package]] name = "hathorlib" -version = "0.14.1" +version = "0.14.0" description = "Hathor Network base objects library" optional = false -python-versions = "<4,>=3.9" +python-versions = ">=3.9,<4" groups = ["main"] -files = [ - {file = "hathorlib-0.14.1-py3-none-any.whl", hash = "sha256:fb234d65ad851bde50e525dbc3c89757825b6d375e384668e8cf99d101d26f98"}, - {file = "hathorlib-0.14.1.tar.gz", hash = "sha256:7ba8a82233ea03ff74a09c12c43c4659209edc6c38af3116e76e829375aeef80"}, -] +files = [] +develop = true [package.dependencies] -aiohttp = {version = ">=3.9.3,<3.10.0", optional = true, markers = "extra == \"client\""} -base58 = ">=2.1.1,<2.2.0" -cryptography = ">=42.0.5,<42.1.0" -pycoin = ">=0.92,<0.93" -pydantic = ">=2.0,<3.0" -pyyaml = ">=6.0.1,<7.0.0" -structlog = {version = ">=22.3.0,<22.4.0", optional = true, markers = "extra == \"client\""} +aiohttp = {version = "~3.9.3", optional = true} +base58 = "~2.1.1" +cryptography = "~42.0.5" +pycoin = "~0.92" +pydantic = "^2.0" +pyyaml = "^6.0.1" +structlog = {version = "~22.3.0", optional = true} [package.extras] client = ["aiohttp (>=3.9.3,<3.10.0)", "structlog (>=22.3.0,<22.4.0)"] +[package.source] +type = "directory" +url = "../hathor-core/hathorlib" + [[package]] name = "idna" version = "3.4" @@ -1463,4 +1465,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.1" python-versions = ">=3.11,<4.0" -content-hash = "c402307f4213d5b744155e3fc0adbeb3ca48ea267615cc4378e0adea8b018225" +content-hash = "cfb73eec5eb83903fb25e89696112f949b02f105e1f88f4efa0131bf4b2442fe" diff --git a/pyproject.toml b/pyproject.toml index c40533f..bbd57c3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ prometheus-client = "^0.9.0" idna_ssl = "^1.1.0" # Cap required: setuptools >=82 removed pkg_resources, which is used by pycoin (a transitive dep via hathorlib) setuptools = ">=68.0,<82" -hathorlib = {version = "^0.14.1", extras = ["client"]} +hathorlib = {path = "../hathor-core/hathorlib", develop = true, extras = ["client"]} python-healthchecklib = "^0.1.0" [tool.poetry.group.dev.dependencies] diff --git a/tests/test_shielded_outputs.py b/tests/test_shielded_outputs.py new file mode 100644 index 0000000..05cea51 --- /dev/null +++ b/tests/test_shielded_outputs.py @@ -0,0 +1,526 @@ +""" +Copyright (c) Hathor Labs and its affiliates. + +This source code is licensed under the MIT license found in the +LICENSE file in the root directory of this source tree. + +Tests for mining transactions with shielded outputs. + +These tests validate that the tx-mining-service can correctly: +1. Parse transactions containing ShieldedOutputsHeader +2. Solve PoW for transactions with shielded outputs (both AmountShielded and FullShielded) +3. Round-trip serialize/deserialize transactions with shielded outputs +4. Mine shielded transactions through the dev-miner HTTP API +""" + +import asyncio +import os +import unittest +from unittest.mock import AsyncMock, MagicMock + +from aiohttp.test_utils import AioHTTPTestCase, unittest_run_loop +from hathorlib import Transaction +from hathorlib.base_transaction import tx_or_block_from_bytes +from hathorlib.headers.shielded_outputs_header import ShieldedOutputsHeader +from hathorlib.transaction.shielded_tx_output import ( + AmountShieldedOutput, + FullShieldedOutput, + OutputMode, + deserialize_shielded_output, + serialize_shielded_output, +) + +import txstratum.time +from txstratum.api import App +from txstratum.dev.manager import DevMiningManager +from txstratum.dev.tx_miner import solve_tx +from txstratum.jobs import TxJob + +# A real serialized transaction (same as test_dev_miner.py) used as a base. +BASE_TX_DATA = bytes.fromhex( + "0001000102000000000000089c0d40a9b1edfb499bc624833fde87ae459d495000393f4aaa00006" + "a473045022100c407d5e8f411f9ae582ebd7acbfcb6ea6170332709fb69acaa34c1b426f1d8f502" + "2003847963768eca9bcdf46e758319fb2699fd28ab657d00f54bef46c37a90405e2103755f2920f" + "f7dc32dc5414cea1cf9e078347f40894caf0c03637d083dbb261c5c000003e800001976a914a04c" + "9e2a0291f53c618fdad2ecb37748efb0eeeb88ac0000151800001976a914545f1156a3b00df622b" + "1d92968c21b962e9d7aa588ac4032a8228c4020c35ed18547020000000047c9881d2bf348d5ffd6" + "ce8398d6bc5d17b3bea75a53c15b7480be950000006ed5794bf69ebe7d7d75e7a0024d98acb85cb" + "9c101b59b8b6073e8667c84e2ee77" +) + +# Fake 33-byte compressed public key (starts with 0x02) +FAKE_COMMITMENT = b'\x02' + b'\xab' * 32 +FAKE_ASSET_COMMITMENT = b'\x03' + b'\xcd' * 32 +FAKE_EPHEMERAL_PUBKEY = b'\x02' + b'\xef' * 32 + +# Fake range proof (~675 bytes in reality, use short placeholder for tests) +FAKE_RANGE_PROOF = os.urandom(675) + +# Fake surjection proof +FAKE_SURJECTION_PROOF = os.urandom(130) + +# Simple P2PKH locking script +FAKE_SCRIPT = bytes.fromhex("76a914a04c9e2a0291f53c618fdad2ecb37748efb0eeeb88ac") + + +def _make_amount_shielded_output(token_data: int = 0) -> AmountShieldedOutput: + """Create a test AmountShieldedOutput.""" + return AmountShieldedOutput( + commitment=FAKE_COMMITMENT, + range_proof=FAKE_RANGE_PROOF, + script=FAKE_SCRIPT, + token_data=token_data, + ephemeral_pubkey=FAKE_EPHEMERAL_PUBKEY, + ) + + +def _make_full_shielded_output() -> FullShieldedOutput: + """Create a test FullShieldedOutput.""" + return FullShieldedOutput( + commitment=FAKE_COMMITMENT, + range_proof=FAKE_RANGE_PROOF, + script=FAKE_SCRIPT, + asset_commitment=FAKE_ASSET_COMMITMENT, + surjection_proof=FAKE_SURJECTION_PROOF, + ephemeral_pubkey=FAKE_EPHEMERAL_PUBKEY, + ) + + +def _build_tx_with_shielded_outputs(shielded_outputs: list) -> bytes: + """Build a serialized transaction with shielded outputs appended as a header. + + Takes the base transaction bytes and appends a ShieldedOutputsHeader. + """ + tx = tx_or_block_from_bytes(BASE_TX_DATA) + assert isinstance(tx, Transaction) + + # Update timestamp to current time + tx.timestamp = int(txstratum.time.time()) + + # Build the shielded outputs header manually by constructing the header + # and attaching it to the transaction. + header = ShieldedOutputsHeader(tx=tx, shielded_outputs=shielded_outputs) + tx.headers = [header] + + # Serialize: this produces funds + graph + nonce + headers + tx.update_hash() + return bytes(tx) + + +# --------------------------------------------------------------------------- +# Shielded output serialization tests +# --------------------------------------------------------------------------- + + +class TestShieldedOutputSerialization(unittest.TestCase): + """Test round-trip serialization of shielded outputs.""" + + __test__ = True + + def test_amount_shielded_roundtrip(self): + """AmountShieldedOutput survives serialize -> deserialize.""" + original = _make_amount_shielded_output() + data = serialize_shielded_output(original) + restored, remaining = deserialize_shielded_output(data) + + self.assertEqual(remaining, b'') + self.assertIsInstance(restored, AmountShieldedOutput) + self.assertEqual(restored.commitment, original.commitment) + self.assertEqual(restored.range_proof, original.range_proof) + self.assertEqual(restored.script, original.script) + self.assertEqual(restored.token_data, original.token_data) + self.assertEqual(restored.ephemeral_pubkey, original.ephemeral_pubkey) + + def test_full_shielded_roundtrip(self): + """FullShieldedOutput survives serialize -> deserialize.""" + original = _make_full_shielded_output() + data = serialize_shielded_output(original) + restored, remaining = deserialize_shielded_output(data) + + self.assertEqual(remaining, b'') + self.assertIsInstance(restored, FullShieldedOutput) + self.assertEqual(restored.commitment, original.commitment) + self.assertEqual(restored.range_proof, original.range_proof) + self.assertEqual(restored.script, original.script) + self.assertEqual(restored.asset_commitment, original.asset_commitment) + self.assertEqual(restored.surjection_proof, original.surjection_proof) + self.assertEqual(restored.ephemeral_pubkey, original.ephemeral_pubkey) + + def test_mode_byte_amount_only(self): + """AmountShieldedOutput serializes with mode byte 0x01.""" + output = _make_amount_shielded_output() + data = serialize_shielded_output(output) + self.assertEqual(data[0], OutputMode.AMOUNT_ONLY) + + def test_mode_byte_fully_shielded(self): + """FullShieldedOutput serializes with mode byte 0x02.""" + output = _make_full_shielded_output() + data = serialize_shielded_output(output) + self.assertEqual(data[0], OutputMode.FULLY_SHIELDED) + + +# --------------------------------------------------------------------------- +# Transaction with shielded outputs: parsing and PoW +# --------------------------------------------------------------------------- + + +class TestShieldedTxParsing(unittest.TestCase): + """Test that transactions with shielded outputs can be parsed and mined.""" + + __test__ = True + + def test_parse_tx_with_amount_shielded_outputs(self): + """A transaction with AmountShieldedOutputs can be serialized and parsed back.""" + outputs = [_make_amount_shielded_output(), _make_amount_shielded_output()] + tx_bytes = _build_tx_with_shielded_outputs(outputs) + + # Parse it back + tx = tx_or_block_from_bytes(tx_bytes) + self.assertIsInstance(tx, Transaction) + self.assertTrue(tx.has_shielded_outputs()) + + header = tx.get_shielded_outputs_header() + self.assertEqual(len(header.shielded_outputs), 2) + for so in header.shielded_outputs: + self.assertIsInstance(so, AmountShieldedOutput) + self.assertEqual(so.commitment, FAKE_COMMITMENT) + + def test_parse_tx_with_full_shielded_outputs(self): + """A transaction with FullShieldedOutputs can be serialized and parsed back.""" + outputs = [_make_full_shielded_output(), _make_full_shielded_output()] + tx_bytes = _build_tx_with_shielded_outputs(outputs) + + tx = tx_or_block_from_bytes(tx_bytes) + self.assertIsInstance(tx, Transaction) + self.assertTrue(tx.has_shielded_outputs()) + + header = tx.get_shielded_outputs_header() + self.assertEqual(len(header.shielded_outputs), 2) + for so in header.shielded_outputs: + self.assertIsInstance(so, FullShieldedOutput) + self.assertEqual(so.asset_commitment, FAKE_ASSET_COMMITMENT) + + def test_parse_tx_with_mixed_shielded_outputs(self): + """A transaction with both AmountShielded and FullShielded outputs parses correctly.""" + outputs = [_make_amount_shielded_output(), _make_full_shielded_output()] + tx_bytes = _build_tx_with_shielded_outputs(outputs) + + tx = tx_or_block_from_bytes(tx_bytes) + self.assertTrue(tx.has_shielded_outputs()) + + header = tx.get_shielded_outputs_header() + self.assertEqual(len(header.shielded_outputs), 2) + self.assertIsInstance(header.shielded_outputs[0], AmountShieldedOutput) + self.assertIsInstance(header.shielded_outputs[1], FullShieldedOutput) + + def test_tx_roundtrip_preserves_bytes(self): + """Serializing and deserializing a shielded tx produces identical bytes.""" + outputs = [_make_amount_shielded_output(), _make_full_shielded_output()] + tx_bytes = _build_tx_with_shielded_outputs(outputs) + + tx = tx_or_block_from_bytes(tx_bytes) + reserialized = bytes(tx) + self.assertEqual(tx_bytes, reserialized) + + def test_solve_tx_with_shielded_outputs_trivial_weight(self): + """solve_tx finds a valid PoW for a transaction with shielded outputs (weight=1).""" + outputs = [_make_amount_shielded_output(), _make_amount_shielded_output()] + tx_bytes = _build_tx_with_shielded_outputs(outputs) + + tx = tx_or_block_from_bytes(tx_bytes) + tx.weight = 1.0 + result = solve_tx(tx) + self.assertTrue(result) + self.assertTrue(tx.verify_pow()) + + def test_solve_tx_with_full_shielded_trivial_weight(self): + """solve_tx works for FullShieldedOutputs too.""" + outputs = [_make_full_shielded_output(), _make_full_shielded_output()] + tx_bytes = _build_tx_with_shielded_outputs(outputs) + + tx = tx_or_block_from_bytes(tx_bytes) + tx.weight = 1.0 + result = solve_tx(tx) + self.assertTrue(result) + self.assertTrue(tx.verify_pow()) + + def test_shielded_outputs_affect_hash(self): + """Adding shielded outputs changes the transaction hash. + + This verifies that the ShieldedOutputsHeader is included in the hash + computation, which is critical for mining correctness. + """ + # Transaction without shielded outputs + tx_plain = tx_or_block_from_bytes(BASE_TX_DATA) + tx_plain.timestamp = int(txstratum.time.time()) + tx_plain.update_hash() + hash_plain = tx_plain.hash + + # Same transaction with shielded outputs + outputs = [_make_amount_shielded_output()] + tx_bytes = _build_tx_with_shielded_outputs(outputs) + tx_shielded = tx_or_block_from_bytes(tx_bytes) + hash_shielded = tx_shielded.hash + + self.assertNotEqual(hash_plain, hash_shielded) + + +# --------------------------------------------------------------------------- +# Dev-miner integration: mine shielded txs through the HTTP API +# --------------------------------------------------------------------------- + + +class TestDevMinerShieldedOutputs(AioHTTPTestCase): + """Test mining transactions with shielded outputs through the dev-miner HTTP API. + + This validates the full lifecycle: submit-job with a shielded tx → poll → done. + """ + + __test__ = True + + async def get_application(self): + self.backend = MagicMock() + self.backend.get_tx_parents = AsyncMock( + return_value=[b"\x00" * 32, b"\x01" * 32] + ) + self.backend.push_tx_or_block = AsyncMock(return_value=True) + self.manager = DevMiningManager(backend=self.backend) + await self.manager.start() + self.healthcheck = MagicMock() + self.myapp = App(self.manager, self.healthcheck) + return self.myapp.app + + @unittest_run_loop + async def test_submit_tx_with_amount_shielded_outputs(self): + """A transaction with AmountShieldedOutputs can be submitted and mined.""" + outputs = [_make_amount_shielded_output(), _make_amount_shielded_output()] + tx_bytes = _build_tx_with_shielded_outputs(outputs) + + # Verify the tx parses as having shielded outputs + tx = tx_or_block_from_bytes(tx_bytes) + self.assertTrue(tx.has_shielded_outputs()) + + resp = await self.client.request( + "POST", "/submit-job", json={"tx": tx_bytes.hex()} + ) + self.assertEqual(200, resp.status) + data = await resp.json() + job_id = data["job_id"] + + # Poll for completion + for _ in range(50): + await asyncio.sleep(0.1) + resp = await self.client.request( + "GET", "/job-status", params={"job-id": job_id} + ) + data = await resp.json() + if data["status"] == "done": + break + + self.assertEqual("done", data["status"]) + self.assertIsNotNone(data["tx"]["nonce"]) + + @unittest_run_loop + async def test_submit_tx_with_full_shielded_outputs(self): + """A transaction with FullShieldedOutputs can be submitted and mined.""" + outputs = [_make_full_shielded_output(), _make_full_shielded_output()] + tx_bytes = _build_tx_with_shielded_outputs(outputs) + + tx = tx_or_block_from_bytes(tx_bytes) + self.assertTrue(tx.has_shielded_outputs()) + + resp = await self.client.request( + "POST", "/submit-job", json={"tx": tx_bytes.hex()} + ) + self.assertEqual(200, resp.status) + data = await resp.json() + job_id = data["job_id"] + + for _ in range(50): + await asyncio.sleep(0.1) + resp = await self.client.request( + "GET", "/job-status", params={"job-id": job_id} + ) + data = await resp.json() + if data["status"] == "done": + break + + self.assertEqual("done", data["status"]) + self.assertIsNotNone(data["tx"]["nonce"]) + + @unittest_run_loop + async def test_submit_tx_with_mixed_shielded_outputs(self): + """A transaction with both output types can be submitted and mined.""" + outputs = [_make_amount_shielded_output(), _make_full_shielded_output()] + tx_bytes = _build_tx_with_shielded_outputs(outputs) + + resp = await self.client.request( + "POST", "/submit-job", json={"tx": tx_bytes.hex()} + ) + self.assertEqual(200, resp.status) + data = await resp.json() + job_id = data["job_id"] + + for _ in range(50): + await asyncio.sleep(0.1) + resp = await self.client.request( + "GET", "/job-status", params={"job-id": job_id} + ) + data = await resp.json() + if data["status"] == "done": + break + + self.assertEqual("done", data["status"]) + + @unittest_run_loop + async def test_mined_shielded_tx_preserves_header(self): + """After mining, the solved transaction still contains the shielded outputs header.""" + outputs = [_make_amount_shielded_output()] + tx_bytes = _build_tx_with_shielded_outputs(outputs) + + resp = await self.client.request( + "POST", "/submit-job", json={"tx": tx_bytes.hex()} + ) + data = await resp.json() + job_id = data["job_id"] + + for _ in range(50): + await asyncio.sleep(0.1) + resp = await self.client.request( + "GET", "/job-status", params={"job-id": job_id} + ) + data = await resp.json() + if data["status"] == "done": + break + + self.assertEqual("done", data["status"]) + + # Retrieve the job from the manager and verify the tx still has shielded outputs + job_uuid = bytes.fromhex(job_id) + job = self.manager.tx_jobs.get(job_uuid) + self.assertIsNotNone(job) + mined_tx = job.get_tx() + self.assertIsInstance(mined_tx, Transaction) + self.assertTrue(mined_tx.has_shielded_outputs()) + + header = mined_tx.get_shielded_outputs_header() + self.assertEqual(len(header.shielded_outputs), 1) + self.assertIsInstance(header.shielded_outputs[0], AmountShieldedOutput) + + @unittest_run_loop + async def test_propagate_shielded_tx(self): + """A shielded tx with propagate=True is pushed to the fullnode after mining.""" + outputs = [_make_amount_shielded_output(), _make_amount_shielded_output()] + tx_bytes = _build_tx_with_shielded_outputs(outputs) + + resp = await self.client.request( + "POST", + "/submit-job", + json={"tx": tx_bytes.hex(), "propagate": True}, + ) + self.assertEqual(200, resp.status) + data = await resp.json() + job_id = data["job_id"] + + for _ in range(50): + await asyncio.sleep(0.1) + resp = await self.client.request( + "GET", "/job-status", params={"job-id": job_id} + ) + data = await resp.json() + if data["status"] == "done": + break + + self.assertEqual("done", data["status"]) + self.backend.push_tx_or_block.assert_called_once() + + @unittest_run_loop + async def test_shielded_tx_with_add_parents(self): + """A shielded tx with add_parents=True fetches parents before mining.""" + outputs = [_make_full_shielded_output(), _make_full_shielded_output()] + tx_bytes = _build_tx_with_shielded_outputs(outputs) + + resp = await self.client.request( + "POST", + "/submit-job", + json={"tx": tx_bytes.hex(), "add_parents": True}, + ) + self.assertEqual(200, resp.status) + data = await resp.json() + job_id = data["job_id"] + + for _ in range(50): + await asyncio.sleep(0.1) + resp = await self.client.request( + "GET", "/job-status", params={"job-id": job_id} + ) + data = await resp.json() + if data["status"] == "done": + break + + self.assertEqual("done", data["status"]) + self.assertEqual(2, len(data["tx"]["parents"])) + + +# --------------------------------------------------------------------------- +# TxJob unit tests with shielded outputs +# --------------------------------------------------------------------------- + + +class TestTxJobShielded(unittest.TestCase): + """Test that TxJob correctly handles transactions with shielded outputs.""" + + __test__ = True + + def test_txjob_parses_shielded_tx(self): + """TxJob can be created from a transaction with shielded outputs.""" + outputs = [_make_amount_shielded_output(), _make_full_shielded_output()] + tx_bytes = _build_tx_with_shielded_outputs(outputs) + + job = TxJob(tx_bytes) + tx = job.get_tx() + self.assertIsInstance(tx, Transaction) + self.assertTrue(tx.has_shielded_outputs()) + + def test_txjob_uuid_includes_shielded_data(self): + """The job UUID (tx hash) differs between shielded and non-shielded versions.""" + # Non-shielded + tx_plain = tx_or_block_from_bytes(BASE_TX_DATA) + tx_plain.timestamp = int(txstratum.time.time()) + tx_plain.update_hash() + job_plain = TxJob(bytes(tx_plain)) + + # Shielded + outputs = [_make_amount_shielded_output()] + tx_bytes = _build_tx_with_shielded_outputs(outputs) + job_shielded = TxJob(tx_bytes) + + self.assertNotEqual(job_plain.uuid, job_shielded.uuid) + + def test_txjob_to_dict_shielded(self): + """to_dict() works correctly for shielded transactions.""" + outputs = [_make_amount_shielded_output()] + tx_bytes = _build_tx_with_shielded_outputs(outputs) + + job = TxJob(tx_bytes) + d = job.to_dict() + self.assertIn("job_id", d) + self.assertIn("status", d) + self.assertEqual(d["status"], "pending") + + def test_txjob_set_parents_shielded(self): + """set_parents works and updates the tx hash for shielded transactions.""" + outputs = [_make_amount_shielded_output()] + tx_bytes = _build_tx_with_shielded_outputs(outputs) + + job = TxJob(tx_bytes) + old_tx_hash = job.get_tx().hash + + new_parents = [b"\xaa" * 32, b"\xbb" * 32] + job.set_parents(new_parents) + + # The internal tx hash should change after parents update + self.assertNotEqual(old_tx_hash, job.get_tx().hash) + self.assertEqual(job.get_tx().parents, new_parents) + # The shielded header should still be present + self.assertTrue(job.get_tx().has_shielded_outputs()) From 844f1f50889fcdd6fbbdcd1b7915851207f86ed1 Mon Sep 17 00:00:00 2001 From: Marcelo Salhab Brogliato Date: Fri, 17 Apr 2026 16:02:43 -0500 Subject: [PATCH 2/3] fix: point hathorlib path to ../hathor-core-4/hathorlib and add large-amount shielded tests Update pyproject.toml, poetry.lock, and Dockerfile so the local hathorlib path dependency resolves to the correct sibling checkout. Add 13 tests covering transparent output amounts across the 4-byte to 8-byte encoding boundary (100M, 2**31-1, 2**31, 100B, 100T, 2**62, MAX_OUTPUT_VALUE) in combination with shielded outputs, exercising round-trip serialization, PoW solving, and full dev-miner HTTP submission. Co-Authored-By: Claude Opus 4.7 (1M context) --- Dockerfile | 10 +- poetry.lock | 6 +- pyproject.toml | 2 +- tests/test_shielded_outputs.py | 190 ++++++++++++++++++++++++++++++++- 4 files changed, 198 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index 017be87..d55f26f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,13 +8,13 @@ RUN apk add --no-cache gcc musl-dev libffi-dev openssl-dev rust cargo RUN pip --no-input --no-cache-dir install --upgrade pip wheel RUN pip --no-input --no-cache-dir install 'poetry>=1.2.0b2' -# Copy hathorlib source (local path dependency: ../hathor-core/hathorlib). +# Copy hathorlib source (local path dependency: ../hathor-core-4/hathorlib). # Build from the parent directory: # docker build -f tx-mining-service/Dockerfile -t tx-mining-service . -COPY hathor-core/hathorlib/ /code/hathor-core/hathorlib/ +COPY hathor-core-4/hathorlib/ /code/hathor-core-4/hathorlib/ -# Install in a subdirectory so the relative path "../hathor-core/hathorlib" -# in pyproject.toml resolves correctly to /code/hathor-core/hathorlib/. +# Install in a subdirectory so the relative path "../hathor-core-4/hathorlib" +# in pyproject.toml resolves correctly to /code/hathor-core-4/hathorlib/. COPY tx-mining-service/poetry.lock tx-mining-service/pyproject.toml /code/tx-mining-service/ WORKDIR /code/tx-mining-service @@ -26,7 +26,7 @@ FROM python:3.11-alpine COPY --from=build /usr/local/lib/python3.11/site-packages /usr/local/lib/python3.11/site-packages # hathorlib is installed in develop mode (.pth file points to this path) -COPY --from=build /code/hathor-core/hathorlib /code/hathor-core/hathorlib +COPY --from=build /code/hathor-core-4/hathorlib /code/hathor-core-4/hathorlib RUN apk add libgcc COPY tx-mining-service/txstratum/ ./txstratum diff --git a/poetry.lock b/poetry.lock index b0c0e44..3090502 100644 --- a/poetry.lock +++ b/poetry.lock @@ -616,7 +616,7 @@ files = [ [[package]] name = "hathorlib" -version = "0.14.0" +version = "0.14.1" description = "Hathor Network base objects library" optional = false python-versions = ">=3.9,<4" @@ -638,7 +638,7 @@ client = ["aiohttp (>=3.9.3,<3.10.0)", "structlog (>=22.3.0,<22.4.0)"] [package.source] type = "directory" -url = "../hathor-core/hathorlib" +url = "../hathor-core-4/hathorlib" [[package]] name = "idna" @@ -1465,4 +1465,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.1" python-versions = ">=3.11,<4.0" -content-hash = "cfb73eec5eb83903fb25e89696112f949b02f105e1f88f4efa0131bf4b2442fe" +content-hash = "63e3481d130e4979fbda00b49b99402dbe3320022be723c163904882fa96a914" diff --git a/pyproject.toml b/pyproject.toml index bbd57c3..155bb8d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ prometheus-client = "^0.9.0" idna_ssl = "^1.1.0" # Cap required: setuptools >=82 removed pkg_resources, which is used by pycoin (a transitive dep via hathorlib) setuptools = ">=68.0,<82" -hathorlib = {path = "../hathor-core/hathorlib", develop = true, extras = ["client"]} +hathorlib = {path = "../hathor-core-4/hathorlib", develop = true, extras = ["client"]} python-healthchecklib = "^0.1.0" [tool.poetry.group.dev.dependencies] diff --git a/tests/test_shielded_outputs.py b/tests/test_shielded_outputs.py index 05cea51..4862aa2 100644 --- a/tests/test_shielded_outputs.py +++ b/tests/test_shielded_outputs.py @@ -86,10 +86,16 @@ def _make_full_shielded_output() -> FullShieldedOutput: ) -def _build_tx_with_shielded_outputs(shielded_outputs: list) -> bytes: +def _build_tx_with_shielded_outputs( + shielded_outputs: list, + transparent_output_value: int | None = None, +) -> bytes: """Build a serialized transaction with shielded outputs appended as a header. Takes the base transaction bytes and appends a ShieldedOutputsHeader. + If transparent_output_value is given, the first transparent output's value + is overwritten with it (useful for testing large amounts, which trigger + the 4-byte -> 8-byte output-value encoding switch at 2**31). """ tx = tx_or_block_from_bytes(BASE_TX_DATA) assert isinstance(tx, Transaction) @@ -97,6 +103,9 @@ def _build_tx_with_shielded_outputs(shielded_outputs: list) -> bytes: # Update timestamp to current time tx.timestamp = int(txstratum.time.time()) + if transparent_output_value is not None: + tx.outputs[0].value = transparent_output_value + # Build the shielded outputs header manually by constructing the header # and attaching it to the transaction. header = ShieldedOutputsHeader(tx=tx, shielded_outputs=shielded_outputs) @@ -524,3 +533,182 @@ def test_txjob_set_parents_shielded(self): self.assertEqual(job.get_tx().parents, new_parents) # The shielded header should still be present self.assertTrue(job.get_tx().has_shielded_outputs()) + + +# --------------------------------------------------------------------------- +# Large transparent amounts combined with shielded outputs +# --------------------------------------------------------------------------- + +# _MAX_OUTPUT_VALUE_32 = 2**31 - 1 is the threshold where output-value encoding +# switches from 4 bytes (signed int32) to 8 bytes (stored as -value in int64). +# MAX_OUTPUT_VALUE = 2**63 is the absolute cap. +SMALL_AMOUNT_4B = 100_000_000 # 100 million, still 4-byte +BOUNDARY_4B = 2**31 - 1 # largest 4-byte value +JUST_OVER_4B = 2**31 # first value forced into 8 bytes +BIG_AMOUNT_8B = 100_000_000_000 # 100 billion, needs 8 bytes +HUGE_AMOUNT_8B = 100_000_000_000_000 # 100 trillion +NEAR_MAX_AMOUNT = 2**62 # well into 8-byte range +MAX_AMOUNT = 2**63 # absolute max permitted + + +class TestShieldedTxLargeAmounts(unittest.TestCase): + """Shielded outputs must coexist with transparent outputs of any legal size. + + The 4B -> 8B output-value encoding switch at 2**31 is the main thing we're + stressing: serialization length changes, but parsing + mining must still + work when a ShieldedOutputsHeader is appended. + """ + + __test__ = True + + def _roundtrip_with_amount(self, amount: int) -> Transaction: + outputs = [_make_amount_shielded_output()] + tx_bytes = _build_tx_with_shielded_outputs( + outputs, transparent_output_value=amount + ) + tx = tx_or_block_from_bytes(tx_bytes) + self.assertIsInstance(tx, Transaction) + self.assertEqual(tx.outputs[0].value, amount) + self.assertTrue(tx.has_shielded_outputs()) + # Bytes round-trip cleanly + self.assertEqual(tx_bytes, bytes(tx)) + return tx + + def test_roundtrip_100_million(self): + """100M (4-byte encoding) survives round-trip with a shielded output.""" + self._roundtrip_with_amount(SMALL_AMOUNT_4B) + + def test_roundtrip_boundary_4byte(self): + """2**31 - 1 (largest 4-byte value) survives round-trip.""" + self._roundtrip_with_amount(BOUNDARY_4B) + + def test_roundtrip_just_over_4byte(self): + """2**31 (first value using 8-byte encoding) survives round-trip.""" + self._roundtrip_with_amount(JUST_OVER_4B) + + def test_roundtrip_100_billion(self): + """100B (8-byte encoding) survives round-trip with a shielded output.""" + self._roundtrip_with_amount(BIG_AMOUNT_8B) + + def test_roundtrip_100_trillion(self): + """100T survives round-trip with a shielded output.""" + self._roundtrip_with_amount(HUGE_AMOUNT_8B) + + def test_roundtrip_near_max(self): + """2**62 survives round-trip with a shielded output.""" + self._roundtrip_with_amount(NEAR_MAX_AMOUNT) + + def test_roundtrip_max_amount(self): + """MAX_OUTPUT_VALUE (2**63) survives round-trip with a shielded output.""" + self._roundtrip_with_amount(MAX_AMOUNT) + + def test_serialized_length_changes_across_4byte_boundary(self): + """Bytes at 2**31 are exactly 4 longer than at 2**31 - 1. + + Sanity check that we're actually hitting the 4B->8B encoding switch. + """ + outputs = [_make_amount_shielded_output()] + small_bytes = _build_tx_with_shielded_outputs( + outputs, transparent_output_value=BOUNDARY_4B + ) + big_bytes = _build_tx_with_shielded_outputs( + outputs, transparent_output_value=JUST_OVER_4B + ) + self.assertEqual(len(big_bytes), len(small_bytes) + 4) + + def test_solve_tx_with_100_billion_amount(self): + """PoW solving works when the tx has a large transparent output + shielded.""" + outputs = [_make_amount_shielded_output(), _make_full_shielded_output()] + tx_bytes = _build_tx_with_shielded_outputs( + outputs, transparent_output_value=BIG_AMOUNT_8B + ) + tx = tx_or_block_from_bytes(tx_bytes) + tx.weight = 1.0 + self.assertTrue(solve_tx(tx)) + self.assertTrue(tx.verify_pow()) + self.assertEqual(tx.outputs[0].value, BIG_AMOUNT_8B) + self.assertTrue(tx.has_shielded_outputs()) + + def test_solve_tx_with_max_amount(self): + """PoW solving works at the absolute max output value.""" + outputs = [_make_full_shielded_output()] + tx_bytes = _build_tx_with_shielded_outputs( + outputs, transparent_output_value=MAX_AMOUNT + ) + tx = tx_or_block_from_bytes(tx_bytes) + tx.weight = 1.0 + self.assertTrue(solve_tx(tx)) + self.assertTrue(tx.verify_pow()) + self.assertEqual(tx.outputs[0].value, MAX_AMOUNT) + + +class TestDevMinerShieldedLargeAmounts(AioHTTPTestCase): + """Mining via the dev-miner HTTP API with large transparent amounts + shielded outputs.""" + + __test__ = True + + async def get_application(self): + self.backend = MagicMock() + self.backend.get_tx_parents = AsyncMock( + return_value=[b"\x00" * 32, b"\x01" * 32] + ) + self.backend.push_tx_or_block = AsyncMock(return_value=True) + self.manager = DevMiningManager(backend=self.backend) + await self.manager.start() + self.healthcheck = MagicMock() + self.myapp = App(self.manager, self.healthcheck) + return self.myapp.app + + async def _submit_and_wait(self, tx_bytes: bytes) -> dict: + resp = await self.client.request( + "POST", "/submit-job", json={"tx": tx_bytes.hex()} + ) + self.assertEqual(200, resp.status) + data = await resp.json() + job_id = data["job_id"] + + for _ in range(50): + await asyncio.sleep(0.1) + resp = await self.client.request( + "GET", "/job-status", params={"job-id": job_id} + ) + data = await resp.json() + if data["status"] == "done": + break + + self.assertEqual("done", data["status"]) + return {"job_id": job_id, "data": data} + + @unittest_run_loop + async def test_submit_tx_100_million(self): + outputs = [_make_amount_shielded_output()] + tx_bytes = _build_tx_with_shielded_outputs( + outputs, transparent_output_value=SMALL_AMOUNT_4B + ) + result = await self._submit_and_wait(tx_bytes) + job = self.manager.tx_jobs.get(bytes.fromhex(result["job_id"])) + self.assertEqual(job.get_tx().outputs[0].value, SMALL_AMOUNT_4B) + + @unittest_run_loop + async def test_submit_tx_100_billion(self): + outputs = [_make_amount_shielded_output(), _make_full_shielded_output()] + tx_bytes = _build_tx_with_shielded_outputs( + outputs, transparent_output_value=BIG_AMOUNT_8B + ) + result = await self._submit_and_wait(tx_bytes) + job = self.manager.tx_jobs.get(bytes.fromhex(result["job_id"])) + mined = job.get_tx() + self.assertEqual(mined.outputs[0].value, BIG_AMOUNT_8B) + self.assertTrue(mined.has_shielded_outputs()) + + @unittest_run_loop + async def test_submit_tx_max_amount(self): + outputs = [_make_full_shielded_output()] + tx_bytes = _build_tx_with_shielded_outputs( + outputs, transparent_output_value=MAX_AMOUNT + ) + result = await self._submit_and_wait(tx_bytes) + job = self.manager.tx_jobs.get(bytes.fromhex(result["job_id"])) + mined = job.get_tx() + self.assertEqual(mined.outputs[0].value, MAX_AMOUNT) + self.assertTrue(mined.has_shielded_outputs()) From 93910ab4b2f3cd1fb4f6742571be727a106442c2 Mon Sep 17 00:00:00 2001 From: Marcelo Salhab Brogliato Date: Thu, 23 Apr 2026 11:58:15 -0500 Subject: [PATCH 3/3] docs: add shielded-outputs docker build and dockerhub push guide Documents the manual build/verify/push procedure for the experimental hathornetwork/tx-mining-service:shielded-outputs-vN image, including the parent-dir build context, the hathorlib prerequisites (ShieldedOutputsHeader and UnshieldBalanceHeader), and the tag-bumping rule. Co-Authored-By: Claude Opus 4.7 (1M context) --- docs/docker-shielded-outputs.md | 72 +++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 docs/docker-shielded-outputs.md diff --git a/docs/docker-shielded-outputs.md b/docs/docker-shielded-outputs.md new file mode 100644 index 0000000..d28ebc5 --- /dev/null +++ b/docs/docker-shielded-outputs.md @@ -0,0 +1,72 @@ +# Building and publishing the `shielded-outputs` Docker image + +This is an **experimental** build that depends on an unreleased hathorlib carrying +shielded-output support. It is not produced by CI — build and push it manually +from this repo. The official CI at `.github/workflows/docker.yml` is unchanged +and still publishes the regular `hathornetwork/tx-mining-service` tags. + +Target image: `hathornetwork/tx-mining-service:shielded-outputs-v1` + +## Prerequisites + +1. hathorlib checkout at the sibling path `../hathor-core-4/hathorlib`, matching + the `hathorlib = { path = "../hathor-core-4/hathorlib", develop = true }` + entry in `pyproject.toml`. +2. That hathorlib must include **both**: + - `hathorlib.headers.ShieldedOutputsHeader` + - `hathorlib.headers.UnshieldBalanceHeader` (`VertexHeaderId.UNSHIELD_BALANCE_HEADER = b'\x13'`), + registered in `hathorlib.vertex_parser.VertexParser.get_supported_headers()`. +3. `docker login` with push access to the `hathornetwork` Docker Hub org. + +Quick check from the host before building: + +``` +python -c "from hathorlib.headers import ShieldedOutputsHeader, UnshieldBalanceHeader" +``` + +## Build + +Build context is the **parent directory** (not this repo), because the +Dockerfile copies `hathor-core-4/hathorlib/` from the sibling path: + +``` +cd /path/to/Hathor # parent of tx-mining-service and hathor-core-4 +docker build \ + -f tx-mining-service/Dockerfile \ + -t tx-mining-service:shielded-outputs-v1 \ + . +``` + +## Verify + +Confirm both headers are importable and registered in the parser before +publishing: + +``` +docker run --rm --entrypoint python tx-mining-service:shielded-outputs-v1 -c " +from hathorlib.headers import ShieldedOutputsHeader, UnshieldBalanceHeader, VertexHeaderId +from hathorlib.vertex_parser import VertexParser +h = VertexParser.get_supported_headers() +assert h[VertexHeaderId.SHIELDED_OUTPUTS_HEADER] is ShieldedOutputsHeader +assert h[VertexHeaderId.UNSHIELD_BALANCE_HEADER] is UnshieldBalanceHeader +print('ok') +" +``` + +## Publish + +``` +docker tag tx-mining-service:shielded-outputs-v1 \ + hathornetwork/tx-mining-service:shielded-outputs-v1 +docker push hathornetwork/tx-mining-service:shielded-outputs-v1 +``` + +Record the pushed digest (`sha256:...`) from the final line of `docker push` +output alongside the hathor-core-4 commit the hathorlib was built from, so the +image is reproducible. + +## Bumping the tag + +When publishing a new experimental cut, increment the numeric suffix +(`shielded-outputs-v2`, `-v3`, ...) rather than overwriting an existing tag — +consumers may be pinned to the old digest.