From 9492b5c5fbfe3d2b364e98604be95f9695770fca Mon Sep 17 00:00:00 2001 From: Elder Millenial Date: Thu, 21 Sep 2023 15:59:59 -0400 Subject: [PATCH 1/9] bugfix: fixed incorrect bytestring encoding for bytestring longer than 64 bytes --- pycardano/plutus.py | 17 ++++++----- pycardano/serialization.py | 55 +++++++++++++++++++++++++++++++++-- test/pycardano/test_plutus.py | 42 +++++++++++++++++++++----- 3 files changed, 97 insertions(+), 17 deletions(-) diff --git a/pycardano/plutus.py b/pycardano/plutus.py index c8562874..508c7437 100644 --- a/pycardano/plutus.py +++ b/pycardano/plutus.py @@ -22,6 +22,7 @@ CBORSerializable, DictCBORSerializable, IndefiniteList, + MetadataIndefiniteList, Primitive, RawCBOR, default_encoder, @@ -489,7 +490,7 @@ def CONSTR_ID(cls): return getattr(cls, k) def __post_init__(self): - valid_types = (PlutusData, dict, IndefiniteList, int, bytes) + valid_types = (PlutusData, dict, MetadataIndefiniteList, int, bytes) for f in fields(self): if inspect.isclass(f.type) and not issubclass(f.type, valid_types): raise TypeError( @@ -499,7 +500,7 @@ def __post_init__(self): def to_shallow_primitive(self) -> CBORTag: primitives: Primitive = super().to_shallow_primitive() if primitives: - primitives = IndefiniteList(primitives) + primitives = MetadataIndefiniteList(primitives) tag = get_tag(self.CONSTR_ID) if tag: return CBORTag(tag, primitives) @@ -553,7 +554,7 @@ def _dfs(obj): return {"int": obj} elif isinstance(obj, bytes): return {"bytes": obj.hex()} - elif isinstance(obj, IndefiniteList) or isinstance(obj, list): + elif isinstance(obj, MetadataIndefiniteList) or isinstance(obj, list): return {"list": [_dfs(item) for item in obj]} elif isinstance(obj, dict): return {"map": [{"v": _dfs(v), "k": _dfs(k)} for k, v in obj.items()]} @@ -669,7 +670,7 @@ def _dfs(obj): elif "bytes" in obj: return bytes.fromhex(obj["bytes"]) elif "list" in obj: - return IndefiniteList([_dfs(item) for item in obj["list"]]) + return MetadataIndefiniteList([_dfs(item) for item in obj["list"]]) else: raise DeserializeException(f"Unexpected data structure: {obj}") else: @@ -701,12 +702,12 @@ class RawPlutusData(CBORSerializable): def to_primitive(self) -> CBORTag: def _dfs(obj): if isinstance(obj, list) and obj: - return IndefiniteList([_dfs(item) for item in obj]) + return MetadataIndefiniteList([_dfs(item) for item in obj]) elif isinstance(obj, dict): return {_dfs(k): _dfs(v) for k, v in obj.items()} elif isinstance(obj, CBORTag) and isinstance(obj.value, list) and obj.value: if obj.tag != 102: - value = IndefiniteList([_dfs(item) for item in obj.value]) + value = MetadataIndefiniteList([_dfs(item) for item in obj.value]) else: value = [_dfs(item) for item in obj.value] return CBORTag(tag=obj.tag, value=value) @@ -723,7 +724,9 @@ def __deepcopy__(self, memo): return self.__class__.from_cbor(self.to_cbor_hex()) -Datum = Union[PlutusData, dict, int, bytes, IndefiniteList, RawCBOR, RawPlutusData] +Datum = Union[ + PlutusData, dict, int, bytes, MetadataIndefiniteList, RawCBOR, RawPlutusData +] """Plutus Datum type. A Union type that contains all valid datum types.""" diff --git a/pycardano/serialization.py b/pycardano/serialization.py index 87aabd42..1577e064 100644 --- a/pycardano/serialization.py +++ b/pycardano/serialization.py @@ -56,6 +56,13 @@ def __init__(self, li: Primitive): # type: ignore super().__init__(li) # type: ignore +class MetadataIndefiniteList(UserList): + """Dummy class to catch special requirements for PlutusData encoding.""" + + def __init__(self, li: Primitive): # type: ignore + super().__init__(li) # type: ignore + + class IndefiniteFrozenList(FrozenList, IndefiniteList): # type: ignore pass @@ -153,6 +160,29 @@ def wrapper(cls, value: Primitive): CBORBase = TypeVar("CBORBase", bound="CBORSerializable") +def plutus_encoder( + encoder: CBOREncoder, value: Union[CBORSerializable, IndefiniteList] +): + """Overload for default_encoder to properly break up bytestrings.""" + if isinstance(value, (IndefiniteList, IndefiniteFrozenList)): + # Currently, cbor2 doesn't support indefinite list, therefore we need special + # handling here to explicitly write header (b'\x9f'), each body item, and footer (b'\xff') to + # the output bytestring. + encoder.write(b"\x9f") + for item in value: + if isinstance(item, bytes) and len(item) > 64: + encoder.write(b"\x5f") + for i in range(0, len(item), 64): + imax = min(i + 64, len(item)) + encoder.encode(item[i:imax]) + encoder.write(b"\xff") + else: + encoder.encode(item) + encoder.write(b"\xff") + else: + default_encoder(encoder, value) + + def default_encoder( encoder: CBOREncoder, value: Union[CBORSerializable, IndefiniteList] ): @@ -166,17 +196,31 @@ def default_encoder( FrozenList, IndefiniteFrozenList, frozendict, + MetadataIndefiniteList, ), ), ( f"Type of input value is not CBORSerializable, " f"got {type(value)} instead." ) - if isinstance(value, (IndefiniteList, IndefiniteFrozenList)): + if isinstance( + value, (IndefiniteList, IndefiniteFrozenList, MetadataIndefiniteList) + ): # Currently, cbor2 doesn't support indefinite list, therefore we need special # handling here to explicitly write header (b'\x9f'), each body item, and footer (b'\xff') to # the output bytestring. encoder.write(b"\x9f") for item in value: - encoder.encode(item) + if ( + isinstance(value, MetadataIndefiniteList) + and isinstance(item, bytes) + and len(item) > 64 + ): + encoder.write(b"\x5f") + for i in range(0, len(item), 64): + imax = min(i + 64, len(item)) + encoder.encode(item[i:imax]) + encoder.write(b"\xff") + else: + encoder.encode(item) encoder.write(b"\xff") elif isinstance(value, RawCBOR): encoder.write(value.cbor) @@ -511,6 +555,13 @@ def _restore_typed_primitive( return IndefiniteList(v) except TypeError: raise DeserializeException(f"Can not initialize IndefiniteList from {v}") + elif isclass(t) and issubclass(t, MetadataIndefiniteList): + try: + return MetadataIndefiniteList(v) + except TypeError: + raise DeserializeException( + f"Can not initialize MetadataIndefiniteList from {v}" + ) elif hasattr(t, "__origin__") and (t.__origin__ is dict): t_args = t.__args__ if len(t_args) != 2: diff --git a/test/pycardano/test_plutus.py b/test/pycardano/test_plutus.py index f2aba0f9..6c3277b9 100644 --- a/test/pycardano/test_plutus.py +++ b/test/pycardano/test_plutus.py @@ -19,7 +19,7 @@ RedeemerTag, plutus_script_hash, ) -from pycardano.serialization import IndefiniteList +from pycardano.serialization import IndefiniteList, MetadataIndefiniteList @dataclass @@ -28,7 +28,7 @@ class MyTest(PlutusData): a: int b: bytes - c: IndefiniteList + c: MetadataIndefiniteList d: dict @@ -76,7 +76,9 @@ def test_plutus_data(): """Ground truth of this test is generated by test/resources/haskell/PlutusData. See its README for more details.""" key_hash = bytes.fromhex("c2ff616e11299d9094ce0a7eb5b7284b705147a822f4ffbd471f971a") deadline = 1643235300000 - testa = BigTest(MyTest(123, b"1234", IndefiniteList([4, 5, 6]), {1: b"1", 2: b"2"})) + testa = BigTest( + MyTest(123, b"1234", MetadataIndefiniteList([4, 5, 6]), {1: b"1", 2: b"2"}) + ) testb = LargestTest() my_vesting = VestingParam( @@ -93,7 +95,9 @@ def test_plutus_data(): def test_plutus_data_json(): key_hash = bytes.fromhex("c2ff616e11299d9094ce0a7eb5b7284b705147a822f4ffbd471f971a") deadline = 1643235300000 - testa = BigTest(MyTest(123, b"1234", IndefiniteList([4, 5, 6]), {1: b"1", 2: b"2"})) + testa = BigTest( + MyTest(123, b"1234", MetadataIndefiniteList([4, 5, 6]), {1: b"1", 2: b"2"}) + ) testb = LargestTest() my_vesting = VestingParam( @@ -159,7 +163,7 @@ def test_plutus_data_cbor_dict(): def test_plutus_data_to_json_wrong_type(): - test = MyTest(123, b"1234", IndefiniteList([4, 5, 6]), {1: b"1", 2: b"2"}) + test = MyTest(123, b"1234", MetadataIndefiniteList([4, 5, 6]), {1: b"1", 2: b"2"}) test.a = "123" with pytest.raises(TypeError): test.to_json() @@ -223,7 +227,7 @@ def test_execution_units_bool(): def test_redeemer(): - data = MyTest(123, b"234", IndefiniteList([4, 5, 6]), {1: b"1", 2: b"2"}) + data = MyTest(123, b"234", MetadataIndefiniteList([4, 5, 6]), {1: b"1", 2: b"2"}) redeemer = MyRedeemer(data, ExecutionUnits(1000000, 1000000)) redeemer.tag = RedeemerTag.SPEND assert ( @@ -234,7 +238,7 @@ def test_redeemer(): def test_redeemer_empty_datum(): - data = MyTest(123, b"234", IndefiniteList([]), {1: b"1", 2: b"2"}) + data = MyTest(123, b"234", MetadataIndefiniteList([]), {1: b"1", 2: b"2"}) redeemer = MyRedeemer(data, ExecutionUnits(1000000, 1000000)) redeemer.tag = RedeemerTag.SPEND assert ( @@ -306,7 +310,7 @@ def test_clone_raw_plutus_data(): def test_clone_plutus_data(): key_hash = bytes.fromhex("c2ff616e11299d9094ce0a7eb5b7284b705147a822f4ffbd471f971a") deadline = 1643235300000 - testa = BigTest(MyTest(123, b"1234", IndefiniteList([4, 5, 6]), {1: b"1", 2: b"2"})) + testa = BigTest(MyTest(123, b"1234", MetadataIndefiniteList([4, 5, 6]), {1: b"1", 2: b"2"})) testb = LargestTest() my_vesting = VestingParam( beneficiary=key_hash, deadline=deadline, testa=testa, testb=testb @@ -396,3 +400,25 @@ class A(PlutusData): assert ( res == res2 ), "Same class has different default constructor id in two consecutive runs" + + +def test_plutus_data_long_bytes(): + @dataclass + class A(PlutusData): + a: bytes + + quote = ( + "The line separating good and evil passes ... right through every human heart." + ) + + quote_hex = ( + "d866821a51e835649f5f5840546865206c696e652073657061726174696e6720676f6f6420616e" + + "64206576696c20706173736573202e2e2e207269676874207468726f7567682065766572794d" + + "2068756d616e2068656172742effff" + ) + + A_tmp = A(quote.encode()) + + assert ( + A_tmp.to_cbor_hex() == quote_hex + ), "Long metadata bytestring is encoded incorrectly." From c92ecfb37162afaa5e189c7953d278fb00612ab6 Mon Sep 17 00:00:00 2001 From: Elder Millenial Date: Thu, 21 Sep 2023 16:01:45 -0400 Subject: [PATCH 2/9] Removed extraneous test encoder --- pycardano/serialization.py | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/pycardano/serialization.py b/pycardano/serialization.py index 1577e064..281ee429 100644 --- a/pycardano/serialization.py +++ b/pycardano/serialization.py @@ -160,29 +160,6 @@ def wrapper(cls, value: Primitive): CBORBase = TypeVar("CBORBase", bound="CBORSerializable") -def plutus_encoder( - encoder: CBOREncoder, value: Union[CBORSerializable, IndefiniteList] -): - """Overload for default_encoder to properly break up bytestrings.""" - if isinstance(value, (IndefiniteList, IndefiniteFrozenList)): - # Currently, cbor2 doesn't support indefinite list, therefore we need special - # handling here to explicitly write header (b'\x9f'), each body item, and footer (b'\xff') to - # the output bytestring. - encoder.write(b"\x9f") - for item in value: - if isinstance(item, bytes) and len(item) > 64: - encoder.write(b"\x5f") - for i in range(0, len(item), 64): - imax = min(i + 64, len(item)) - encoder.encode(item[i:imax]) - encoder.write(b"\xff") - else: - encoder.encode(item) - encoder.write(b"\xff") - else: - default_encoder(encoder, value) - - def default_encoder( encoder: CBOREncoder, value: Union[CBORSerializable, IndefiniteList] ): From 6dbdd4fdc14bc3bd35779dc85ce61e185856e23d Mon Sep 17 00:00:00 2001 From: Elder Millenial Date: Thu, 28 Sep 2023 08:15:56 -0400 Subject: [PATCH 3/9] Removed metadata dummy class --- pycardano/plutus.py | 17 +++++++---------- pycardano/serialization.py | 25 ++----------------------- test/pycardano/test_plutus.py | 20 ++++++++------------ 3 files changed, 17 insertions(+), 45 deletions(-) diff --git a/pycardano/plutus.py b/pycardano/plutus.py index 508c7437..c8562874 100644 --- a/pycardano/plutus.py +++ b/pycardano/plutus.py @@ -22,7 +22,6 @@ CBORSerializable, DictCBORSerializable, IndefiniteList, - MetadataIndefiniteList, Primitive, RawCBOR, default_encoder, @@ -490,7 +489,7 @@ def CONSTR_ID(cls): return getattr(cls, k) def __post_init__(self): - valid_types = (PlutusData, dict, MetadataIndefiniteList, int, bytes) + valid_types = (PlutusData, dict, IndefiniteList, int, bytes) for f in fields(self): if inspect.isclass(f.type) and not issubclass(f.type, valid_types): raise TypeError( @@ -500,7 +499,7 @@ def __post_init__(self): def to_shallow_primitive(self) -> CBORTag: primitives: Primitive = super().to_shallow_primitive() if primitives: - primitives = MetadataIndefiniteList(primitives) + primitives = IndefiniteList(primitives) tag = get_tag(self.CONSTR_ID) if tag: return CBORTag(tag, primitives) @@ -554,7 +553,7 @@ def _dfs(obj): return {"int": obj} elif isinstance(obj, bytes): return {"bytes": obj.hex()} - elif isinstance(obj, MetadataIndefiniteList) or isinstance(obj, list): + elif isinstance(obj, IndefiniteList) or isinstance(obj, list): return {"list": [_dfs(item) for item in obj]} elif isinstance(obj, dict): return {"map": [{"v": _dfs(v), "k": _dfs(k)} for k, v in obj.items()]} @@ -670,7 +669,7 @@ def _dfs(obj): elif "bytes" in obj: return bytes.fromhex(obj["bytes"]) elif "list" in obj: - return MetadataIndefiniteList([_dfs(item) for item in obj["list"]]) + return IndefiniteList([_dfs(item) for item in obj["list"]]) else: raise DeserializeException(f"Unexpected data structure: {obj}") else: @@ -702,12 +701,12 @@ class RawPlutusData(CBORSerializable): def to_primitive(self) -> CBORTag: def _dfs(obj): if isinstance(obj, list) and obj: - return MetadataIndefiniteList([_dfs(item) for item in obj]) + return IndefiniteList([_dfs(item) for item in obj]) elif isinstance(obj, dict): return {_dfs(k): _dfs(v) for k, v in obj.items()} elif isinstance(obj, CBORTag) and isinstance(obj.value, list) and obj.value: if obj.tag != 102: - value = MetadataIndefiniteList([_dfs(item) for item in obj.value]) + value = IndefiniteList([_dfs(item) for item in obj.value]) else: value = [_dfs(item) for item in obj.value] return CBORTag(tag=obj.tag, value=value) @@ -724,9 +723,7 @@ def __deepcopy__(self, memo): return self.__class__.from_cbor(self.to_cbor_hex()) -Datum = Union[ - PlutusData, dict, int, bytes, MetadataIndefiniteList, RawCBOR, RawPlutusData -] +Datum = Union[PlutusData, dict, int, bytes, IndefiniteList, RawCBOR, RawPlutusData] """Plutus Datum type. A Union type that contains all valid datum types.""" diff --git a/pycardano/serialization.py b/pycardano/serialization.py index 281ee429..ffa19783 100644 --- a/pycardano/serialization.py +++ b/pycardano/serialization.py @@ -56,13 +56,6 @@ def __init__(self, li: Primitive): # type: ignore super().__init__(li) # type: ignore -class MetadataIndefiniteList(UserList): - """Dummy class to catch special requirements for PlutusData encoding.""" - - def __init__(self, li: Primitive): # type: ignore - super().__init__(li) # type: ignore - - class IndefiniteFrozenList(FrozenList, IndefiniteList): # type: ignore pass @@ -173,24 +166,17 @@ def default_encoder( FrozenList, IndefiniteFrozenList, frozendict, - MetadataIndefiniteList, ), ), ( f"Type of input value is not CBORSerializable, " f"got {type(value)} instead." ) - if isinstance( - value, (IndefiniteList, IndefiniteFrozenList, MetadataIndefiniteList) - ): + if isinstance(value, (IndefiniteList, IndefiniteFrozenList)): # Currently, cbor2 doesn't support indefinite list, therefore we need special # handling here to explicitly write header (b'\x9f'), each body item, and footer (b'\xff') to # the output bytestring. encoder.write(b"\x9f") for item in value: - if ( - isinstance(value, MetadataIndefiniteList) - and isinstance(item, bytes) - and len(item) > 64 - ): + if isinstance(item, bytes) and len(item) > 64: encoder.write(b"\x5f") for i in range(0, len(item), 64): imax = min(i + 64, len(item)) @@ -532,13 +518,6 @@ def _restore_typed_primitive( return IndefiniteList(v) except TypeError: raise DeserializeException(f"Can not initialize IndefiniteList from {v}") - elif isclass(t) and issubclass(t, MetadataIndefiniteList): - try: - return MetadataIndefiniteList(v) - except TypeError: - raise DeserializeException( - f"Can not initialize MetadataIndefiniteList from {v}" - ) elif hasattr(t, "__origin__") and (t.__origin__ is dict): t_args = t.__args__ if len(t_args) != 2: diff --git a/test/pycardano/test_plutus.py b/test/pycardano/test_plutus.py index 6c3277b9..984e6e3e 100644 --- a/test/pycardano/test_plutus.py +++ b/test/pycardano/test_plutus.py @@ -19,7 +19,7 @@ RedeemerTag, plutus_script_hash, ) -from pycardano.serialization import IndefiniteList, MetadataIndefiniteList +from pycardano.serialization import IndefiniteList @dataclass @@ -28,7 +28,7 @@ class MyTest(PlutusData): a: int b: bytes - c: MetadataIndefiniteList + c: IndefiniteList d: dict @@ -76,9 +76,7 @@ def test_plutus_data(): """Ground truth of this test is generated by test/resources/haskell/PlutusData. See its README for more details.""" key_hash = bytes.fromhex("c2ff616e11299d9094ce0a7eb5b7284b705147a822f4ffbd471f971a") deadline = 1643235300000 - testa = BigTest( - MyTest(123, b"1234", MetadataIndefiniteList([4, 5, 6]), {1: b"1", 2: b"2"}) - ) + testa = BigTest(MyTest(123, b"1234", IndefiniteList([4, 5, 6]), {1: b"1", 2: b"2"})) testb = LargestTest() my_vesting = VestingParam( @@ -95,9 +93,7 @@ def test_plutus_data(): def test_plutus_data_json(): key_hash = bytes.fromhex("c2ff616e11299d9094ce0a7eb5b7284b705147a822f4ffbd471f971a") deadline = 1643235300000 - testa = BigTest( - MyTest(123, b"1234", MetadataIndefiniteList([4, 5, 6]), {1: b"1", 2: b"2"}) - ) + testa = BigTest(MyTest(123, b"1234", IndefiniteList([4, 5, 6]), {1: b"1", 2: b"2"})) testb = LargestTest() my_vesting = VestingParam( @@ -163,7 +159,7 @@ def test_plutus_data_cbor_dict(): def test_plutus_data_to_json_wrong_type(): - test = MyTest(123, b"1234", MetadataIndefiniteList([4, 5, 6]), {1: b"1", 2: b"2"}) + test = MyTest(123, b"1234", IndefiniteList([4, 5, 6]), {1: b"1", 2: b"2"}) test.a = "123" with pytest.raises(TypeError): test.to_json() @@ -227,7 +223,7 @@ def test_execution_units_bool(): def test_redeemer(): - data = MyTest(123, b"234", MetadataIndefiniteList([4, 5, 6]), {1: b"1", 2: b"2"}) + data = MyTest(123, b"234", IndefiniteList([4, 5, 6]), {1: b"1", 2: b"2"}) redeemer = MyRedeemer(data, ExecutionUnits(1000000, 1000000)) redeemer.tag = RedeemerTag.SPEND assert ( @@ -238,7 +234,7 @@ def test_redeemer(): def test_redeemer_empty_datum(): - data = MyTest(123, b"234", MetadataIndefiniteList([]), {1: b"1", 2: b"2"}) + data = MyTest(123, b"234", IndefiniteList([]), {1: b"1", 2: b"2"}) redeemer = MyRedeemer(data, ExecutionUnits(1000000, 1000000)) redeemer.tag = RedeemerTag.SPEND assert ( @@ -310,7 +306,7 @@ def test_clone_raw_plutus_data(): def test_clone_plutus_data(): key_hash = bytes.fromhex("c2ff616e11299d9094ce0a7eb5b7284b705147a822f4ffbd471f971a") deadline = 1643235300000 - testa = BigTest(MyTest(123, b"1234", MetadataIndefiniteList([4, 5, 6]), {1: b"1", 2: b"2"})) + testa = BigTest(MyTest(123, b"1234", IndefiniteList([4, 5, 6]), {1: b"1", 2: b"2"})) testb = LargestTest() my_vesting = VestingParam( beneficiary=key_hash, deadline=deadline, testa=testa, testb=testb From b663a3bfdd6aed54461d18112a9f8bd2946b849b Mon Sep 17 00:00:00 2001 From: Elder Millenial Date: Wed, 4 Oct 2023 22:53:20 -0400 Subject: [PATCH 4/9] Added dummy ByteString class --- pycardano/serialization.py | 30 ++++++++++++++++++++++-------- test/pycardano/test_plutus.py | 23 +++++++++++++++++++++-- 2 files changed, 43 insertions(+), 10 deletions(-) diff --git a/pycardano/serialization.py b/pycardano/serialization.py index ffa19783..988524c9 100644 --- a/pycardano/serialization.py +++ b/pycardano/serialization.py @@ -5,6 +5,7 @@ import re import typing from collections import OrderedDict, UserList, defaultdict +from collections.abc import Sequence from copy import deepcopy from dataclasses import Field, dataclass, fields from datetime import datetime @@ -60,6 +61,14 @@ class IndefiniteFrozenList(FrozenList, IndefiniteList): # type: ignore pass +@dataclass +class ByteString: + value: bytes + + def __hash__(self): + return hash(self.value) + + @dataclass class RawCBOR: """A wrapper class for bytes that represents a CBOR value.""" @@ -160,6 +169,7 @@ def default_encoder( assert isinstance( value, ( + ByteString, CBORSerializable, IndefiniteList, RawCBOR, @@ -176,15 +186,17 @@ def default_encoder( # the output bytestring. encoder.write(b"\x9f") for item in value: - if isinstance(item, bytes) and len(item) > 64: - encoder.write(b"\x5f") - for i in range(0, len(item), 64): - imax = min(i + 64, len(item)) - encoder.encode(item[i:imax]) - encoder.write(b"\xff") - else: - encoder.encode(item) + encoder.encode(item) encoder.write(b"\xff") + elif isinstance(value, ByteString): + if len(value.value) > 64: + encoder.write(b"\x5f") + for i in range(0, len(value.value), 64): + imax = min(i + 64, len(value.value)) + encoder.encode(value.value[i:imax]) + encoder.write(b"\xff") + else: + encoder.encode(value.value) elif isinstance(value, RawCBOR): encoder.write(value.cbor) elif isinstance(value, FrozenList): @@ -247,6 +259,8 @@ def to_primitive(self) -> Primitive: def _dfs(value, freeze=False): if isinstance(value, CBORSerializable): return _dfs(value.to_primitive(), freeze) + elif isinstance(value, bytes): + return ByteString(value) elif isinstance(value, (dict, OrderedDict, defaultdict)): _dict = type(value)() if hasattr(value, "default_factory"): diff --git a/test/pycardano/test_plutus.py b/test/pycardano/test_plutus.py index 984e6e3e..5138cb7c 100644 --- a/test/pycardano/test_plutus.py +++ b/test/pycardano/test_plutus.py @@ -245,7 +245,7 @@ def test_redeemer_empty_datum(): def test_cost_model(): - assert ( + print( "a141005901d59f1a000302590001011a00060bc719026d00011a000249f01903e800011" "a000249f018201a0025cea81971f70419744d186419744d186419744d186419744d1864" "19744d186419744d18641864186419744d18641a000249f018201a000249f018201a000" @@ -259,7 +259,26 @@ def test_cost_model(): "7e2318760001011a000242201a00067e2318760001011a0025cea81971f704001a00014" "1bb041a000249f019138800011a000249f018201a000302590001011a000249f018201a" "000249f018201a000249f018201a000249f018201a000249f018201a000249f018201a0" - "00249f018201a00330da70101ff" == COST_MODELS.to_cbor_hex() + "00249f018201a00330da70101ff" + ) + print() + print(COST_MODELS.to_cbor_hex()) + assert ( + "a141005f58409f1a000302590001011a00060bc719026d00011a000249f01903e800011" + "a000249f018201a0025cea81971f70419744d186419744d186419744d186419744d1858" + "406419744d186419744d18641864186419744d18641a000249f018201a000249f018201" + "a000249f018201a000249f01903e800011a000249f018201a000249f019584003e80008" + "1a000242201a00067e2318760001011a000249f01903e800081a000249f01a0001b7981" + "8f7011a000249f0192710011a0002155e19052e011903e81a5840000249f01903e8011a" + "000249f018201a000249f018201a000249f0182001011a000249f0011a000249f0041a0" + "00194af18f8011a000194af18f8011a0002377c5840190556011a0002bdea1901f1011a" + "000249f018201a000249f018201a000249f018201a000249f018201a000249f018201a0" + "00249f018201a000242201a00067e584023187600010119f04c192bd200011a000249f0" + "18201a000242201a00067e2318760001011a000242201a00067e2318760001011a0025c" + "ea81971f704001a0001584041bb041a000249f019138800011a000249f018201a000302" + "590001011a000249f018201a000249f018201a000249f018201a000249f018201a00024" + "9f018201a55000249f018201a000249f018201a00330da70101ffff" + == COST_MODELS.to_cbor_hex() ) From 364ba67292174ff60752a470075073a6e440b738 Mon Sep 17 00:00:00 2001 From: Elder Millenial Date: Wed, 4 Oct 2023 23:00:43 -0400 Subject: [PATCH 5/9] Added ByteString equality test for bytes --- pycardano/serialization.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pycardano/serialization.py b/pycardano/serialization.py index 988524c9..d2dcfcb1 100644 --- a/pycardano/serialization.py +++ b/pycardano/serialization.py @@ -68,6 +68,12 @@ class ByteString: def __hash__(self): return hash(self.value) + def __eq__(self, other: Union[bytes, ByteString]): + if isinstance(other, ByteString): + return self.value == other.value + else: + return self.value == other + @dataclass class RawCBOR: From 5a90438fe788e6899de092eea357c7e01d29d5f3 Mon Sep 17 00:00:00 2001 From: Elder Millenial Date: Thu, 5 Oct 2023 08:45:14 -0400 Subject: [PATCH 6/9] Updated test reference hash --- test/pycardano/test_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/pycardano/test_util.py b/test/pycardano/test_util.py index 118c8e13..8e7fecee 100644 --- a/test/pycardano/test_util.py +++ b/test/pycardano/test_util.py @@ -149,7 +149,7 @@ def test_script_data_hash(): redeemers = [Redeemer(unit, ExecutionUnits(1000000, 1000000))] redeemers[0].tag = RedeemerTag.SPEND assert ScriptDataHash.from_primitive( - "032d812ee0731af78fe4ec67e4d30d16313c09e6fb675af28f825797e8b5621d" + "b11ed6f6046df925b6409b850ac54a829cd1e7603145c9aaf765885d8ec64da7" ) == script_data_hash(redeemers=redeemers, datums=[unit]) From c71dd821016e25c2e6b83f2897504e4fd29fe882 Mon Sep 17 00:00:00 2001 From: Elder Millenial Date: Tue, 10 Oct 2023 08:14:26 -0400 Subject: [PATCH 7/9] Updated byte encoding errors, added ByteString to as valid PlutusData type --- pycardano/metadata.py | 1 + pycardano/plutus.py | 22 +++++++++++++++++++--- pycardano/serialization.py | 3 --- test/pycardano/test_plutus.py | 35 ++++++++--------------------------- test/pycardano/test_util.py | 2 +- 5 files changed, 29 insertions(+), 34 deletions(-) diff --git a/pycardano/metadata.py b/pycardano/metadata.py index 7a280017..6ddfa708 100644 --- a/pycardano/metadata.py +++ b/pycardano/metadata.py @@ -41,6 +41,7 @@ def _validate_type_and_size(data): if len(data) > self.MAX_ITEM_SIZE: raise InvalidArgumentException( f"The size of {data} exceeds {self.MAX_ITEM_SIZE} bytes." + "Use pycardano.serialization.ByteString for long bytes." ) elif isinstance(data, str): if len(data.encode("utf-8")) > self.MAX_ITEM_SIZE: diff --git a/pycardano/plutus.py b/pycardano/plutus.py index c8562874..eee7a297 100644 --- a/pycardano/plutus.py +++ b/pycardano/plutus.py @@ -14,11 +14,12 @@ from nacl.encoding import RawEncoder from nacl.hash import blake2b -from pycardano.exception import DeserializeException +from pycardano.exception import DeserializeException, InvalidArgumentException from pycardano.hash import DATUM_HASH_SIZE, SCRIPT_HASH_SIZE, DatumHash, ScriptHash from pycardano.nativescript import NativeScript from pycardano.serialization import ( ArrayCBORSerializable, + ByteString, CBORSerializable, DictCBORSerializable, IndefiniteList, @@ -468,6 +469,8 @@ class will reduce the complexity of serialization and deserialization tremendous >>> assert test == Test.from_cbor("d87a9f187b43333231ff") """ + MAX_BYTES_SIZE = 64 + @classproperty def CONSTR_ID(cls): """ @@ -483,19 +486,27 @@ def CONSTR_ID(cls): + "*" + "*".join([f"{f.name}~{f.type}" for f in fields(cls)]) ) + print(det_string) det_hash = sha256(det_string.encode("utf8")).hexdigest() setattr(cls, k, int(det_hash, 16) % 2**32) return getattr(cls, k) def __post_init__(self): - valid_types = (PlutusData, dict, IndefiniteList, int, bytes) + valid_types = (PlutusData, dict, IndefiniteList, int, ByteString, bytes) for f in fields(self): if inspect.isclass(f.type) and not issubclass(f.type, valid_types): raise TypeError( f"Invalid field type: {f.type}. A field in PlutusData should be one of {valid_types}" ) + data = getattr(self, f.name) + if isinstance(data, bytes) and len(data) > 64: + raise InvalidArgumentException( + f"The size of {data} exceeds {self.MAX_BYTES_SIZE} bytes. " + "Use pycardano.serialization.ByteString for long bytes." + ) + def to_shallow_primitive(self) -> CBORTag: primitives: Primitive = super().to_shallow_primitive() if primitives: @@ -553,6 +564,8 @@ def _dfs(obj): return {"int": obj} elif isinstance(obj, bytes): return {"bytes": obj.hex()} + elif isinstance(obj, ByteString): + return {"bytes": obj.value.hex()} elif isinstance(obj, IndefiniteList) or isinstance(obj, list): return {"list": [_dfs(item) for item in obj]} elif isinstance(obj, dict): @@ -667,7 +680,10 @@ def _dfs(obj): elif "int" in obj: return obj["int"] elif "bytes" in obj: - return bytes.fromhex(obj["bytes"]) + if len(obj["bytes"]) > 64: + return ByteString(bytes.fromhex(obj["bytes"])) + else: + return bytes.fromhex(obj["bytes"]) elif "list" in obj: return IndefiniteList([_dfs(item) for item in obj["list"]]) else: diff --git a/pycardano/serialization.py b/pycardano/serialization.py index d2dcfcb1..fd3a7495 100644 --- a/pycardano/serialization.py +++ b/pycardano/serialization.py @@ -5,7 +5,6 @@ import re import typing from collections import OrderedDict, UserList, defaultdict -from collections.abc import Sequence from copy import deepcopy from dataclasses import Field, dataclass, fields from datetime import datetime @@ -265,8 +264,6 @@ def to_primitive(self) -> Primitive: def _dfs(value, freeze=False): if isinstance(value, CBORSerializable): return _dfs(value.to_primitive(), freeze) - elif isinstance(value, bytes): - return ByteString(value) elif isinstance(value, (dict, OrderedDict, defaultdict)): _dict = type(value)() if hasattr(value, "default_factory"): diff --git a/test/pycardano/test_plutus.py b/test/pycardano/test_plutus.py index 5138cb7c..ba3cd992 100644 --- a/test/pycardano/test_plutus.py +++ b/test/pycardano/test_plutus.py @@ -19,7 +19,7 @@ RedeemerTag, plutus_script_hash, ) -from pycardano.serialization import IndefiniteList +from pycardano.serialization import ByteString, IndefiniteList @dataclass @@ -245,7 +245,7 @@ def test_redeemer_empty_datum(): def test_cost_model(): - print( + assert ( "a141005901d59f1a000302590001011a00060bc719026d00011a000249f01903e800011" "a000249f018201a0025cea81971f70419744d186419744d186419744d186419744d1864" "19744d186419744d18641864186419744d18641a000249f018201a000249f018201a000" @@ -259,26 +259,7 @@ def test_cost_model(): "7e2318760001011a000242201a00067e2318760001011a0025cea81971f704001a00014" "1bb041a000249f019138800011a000249f018201a000302590001011a000249f018201a" "000249f018201a000249f018201a000249f018201a000249f018201a000249f018201a0" - "00249f018201a00330da70101ff" - ) - print() - print(COST_MODELS.to_cbor_hex()) - assert ( - "a141005f58409f1a000302590001011a00060bc719026d00011a000249f01903e800011" - "a000249f018201a0025cea81971f70419744d186419744d186419744d186419744d1858" - "406419744d186419744d18641864186419744d18641a000249f018201a000249f018201" - "a000249f018201a000249f01903e800011a000249f018201a000249f019584003e80008" - "1a000242201a00067e2318760001011a000249f01903e800081a000249f01a0001b7981" - "8f7011a000249f0192710011a0002155e19052e011903e81a5840000249f01903e8011a" - "000249f018201a000249f018201a000249f0182001011a000249f0011a000249f0041a0" - "00194af18f8011a000194af18f8011a0002377c5840190556011a0002bdea1901f1011a" - "000249f018201a000249f018201a000249f018201a000249f018201a000249f018201a0" - "00249f018201a000242201a00067e584023187600010119f04c192bd200011a000249f0" - "18201a000242201a00067e2318760001011a000242201a00067e2318760001011a0025c" - "ea81971f704001a0001584041bb041a000249f019138800011a000249f018201a000302" - "590001011a000249f018201a000249f018201a000249f018201a000249f018201a00024" - "9f018201a55000249f018201a000249f018201a00330da70101ffff" - == COST_MODELS.to_cbor_hex() + "00249f018201a00330da70101ff" == COST_MODELS.to_cbor_hex() ) @@ -420,19 +401,19 @@ class A(PlutusData): def test_plutus_data_long_bytes(): @dataclass class A(PlutusData): - a: bytes + a: ByteString quote = ( "The line separating good and evil passes ... right through every human heart." ) quote_hex = ( - "d866821a51e835649f5f5840546865206c696e652073657061726174696e6720676f6f6420616e" - + "64206576696c20706173736573202e2e2e207269676874207468726f7567682065766572794d" - + "2068756d616e2068656172742effff" + "d866821a8e5890cf9f5f5840546865206c696e652073657061726174696e6720676f6f6420616" + "e64206576696c20706173736573202e2e2e207269676874207468726f7567682065766572794d" + "2068756d616e2068656172742effff" ) - A_tmp = A(quote.encode()) + A_tmp = A(ByteString(quote.encode())) assert ( A_tmp.to_cbor_hex() == quote_hex diff --git a/test/pycardano/test_util.py b/test/pycardano/test_util.py index 8e7fecee..118c8e13 100644 --- a/test/pycardano/test_util.py +++ b/test/pycardano/test_util.py @@ -149,7 +149,7 @@ def test_script_data_hash(): redeemers = [Redeemer(unit, ExecutionUnits(1000000, 1000000))] redeemers[0].tag = RedeemerTag.SPEND assert ScriptDataHash.from_primitive( - "b11ed6f6046df925b6409b850ac54a829cd1e7603145c9aaf765885d8ec64da7" + "032d812ee0731af78fe4ec67e4d30d16313c09e6fb675af28f825797e8b5621d" ) == script_data_hash(redeemers=redeemers, datums=[unit]) From 945e52ad3bef4fb94fd517d2c77522f3489c8f8b Mon Sep 17 00:00:00 2001 From: Elder Millenial Date: Tue, 10 Oct 2023 08:16:28 -0400 Subject: [PATCH 8/9] Removed debug print statements --- pycardano/plutus.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pycardano/plutus.py b/pycardano/plutus.py index eee7a297..3ee15800 100644 --- a/pycardano/plutus.py +++ b/pycardano/plutus.py @@ -486,7 +486,6 @@ def CONSTR_ID(cls): + "*" + "*".join([f"{f.name}~{f.type}" for f in fields(cls)]) ) - print(det_string) det_hash = sha256(det_string.encode("utf8")).hexdigest() setattr(cls, k, int(det_hash, 16) % 2**32) From 81abc6a8aaf889fec53aa487fbbef2c8f835e693 Mon Sep 17 00:00:00 2001 From: Elder Millenial Date: Wed, 11 Oct 2023 09:08:28 -0400 Subject: [PATCH 9/9] Fixed mypy error for equality checks on ByteString --- pycardano/serialization.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pycardano/serialization.py b/pycardano/serialization.py index fd3a7495..7b302e04 100644 --- a/pycardano/serialization.py +++ b/pycardano/serialization.py @@ -67,11 +67,13 @@ class ByteString: def __hash__(self): return hash(self.value) - def __eq__(self, other: Union[bytes, ByteString]): + def __eq__(self, other: object): if isinstance(other, ByteString): return self.value == other.value - else: + elif isinstance(other, bytes): return self.value == other + else: + return False @dataclass