diff --git a/.test/lookup-test.json b/.test/lookup-test.json index ae83593..44a86b4 100644 --- a/.test/lookup-test.json +++ b/.test/lookup-test.json @@ -1,79 +1,374 @@ [ - { - "schemaVersion": 2, - "mediaType": "application/vnd.docker.distribution.manifest.list.v2+json", - "manifests": [ - { - "mediaType": "application/vnd.docker.distribution.manifest.v2+json", - "digest": "sha256:2f19ce27632e6baf4ebb1b582960d68948e52902c8cfac10133da0058f1dab23", - "size": 946, - "annotations": { - "com.docker.official-images.bashbrew.arch": "windows-amd64", - "org.opencontainers.image.ref.name": "tianon/test@sha256:2f19ce27632e6baf4ebb1b582960d68948e52902c8cfac10133da0058f1dab23" - }, - "platform": { - "architecture": "amd64", - "os": "windows", - "os.version": "10.0.20348.2340" + [ + [ + "tianon/test@sha256:2f19ce27632e6baf4ebb1b582960d68948e52902c8cfac10133da0058f1dab23" + ], + { + "schemaVersion": 2, + "mediaType": "application/vnd.docker.distribution.manifest.list.v2+json", + "manifests": [ + { + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "digest": "sha256:2f19ce27632e6baf4ebb1b582960d68948e52902c8cfac10133da0058f1dab23", + "size": 946, + "annotations": { + "com.docker.official-images.bashbrew.arch": "windows-amd64", + "org.opencontainers.image.ref.name": "tianon/test@sha256:2f19ce27632e6baf4ebb1b582960d68948e52902c8cfac10133da0058f1dab23" + }, + "platform": { + "architecture": "amd64", + "os": "windows", + "os.version": "10.0.20348.2340" + } } + ], + "annotations": { + "org.opencontainers.image.ref.name": "tianon/test@sha256:2f19ce27632e6baf4ebb1b582960d68948e52902c8cfac10133da0058f1dab23" } + } + ], + [ + [ + "tianon/test@sha256:347290ddd775c1b85a3e381b09edde95242478eb65153e9b17225356f4c072ac" ], - "annotations": { - "org.opencontainers.image.ref.name": "tianon/test@sha256:2f19ce27632e6baf4ebb1b582960d68948e52902c8cfac10133da0058f1dab23" + { + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.index.v1+json", + "manifests": [ + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:e2fc4e5012d16e7fe466f5291c476431beaa1f9b90a5c2125b493ed28e2aba57", + "size": 861, + "annotations": { + "com.docker.official-images.bashbrew.arch": "amd64", + "org.opencontainers.image.ref.name": "tianon/test@sha256:e2fc4e5012d16e7fe466f5291c476431beaa1f9b90a5c2125b493ed28e2aba57", + "org.opencontainers.image.revision": "3fb6ebca4163bf5b9cc496ac3e8f11cb1e754aee", + "org.opencontainers.image.source": "https://github.com/docker-library/hello-world.git#3fb6ebca4163bf5b9cc496ac3e8f11cb1e754aee:amd64/hello-world", + "org.opencontainers.image.url": "https://hub.docker.com/_/hello-world", + "org.opencontainers.image.version": "linux" + }, + "platform": { + "architecture": "amd64", + "os": "linux" + } + }, + { + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "digest": "sha256:2f19ce27632e6baf4ebb1b582960d68948e52902c8cfac10133da0058f1dab23", + "size": 946, + "annotations": { + "com.docker.official-images.bashbrew.arch": "windows-amd64", + "org.opencontainers.image.ref.name": "tianon/test@sha256:2f19ce27632e6baf4ebb1b582960d68948e52902c8cfac10133da0058f1dab23" + }, + "platform": { + "architecture": "amd64", + "os": "windows", + "os.version": "10.0.20348.2340" + } + }, + { + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "digest": "sha256:3a0bd0fb5ad6dd6528dc78726b3df78e980b39b379e99c5a508904ec17cfafe5", + "size": 946, + "annotations": { + "com.docker.official-images.bashbrew.arch": "windows-amd64", + "org.opencontainers.image.ref.name": "tianon/test@sha256:3a0bd0fb5ad6dd6528dc78726b3df78e980b39b379e99c5a508904ec17cfafe5" + }, + "platform": { + "architecture": "amd64", + "os": "windows", + "os.version": "10.0.17763.5576" + } + } + ], + "annotations": { + "org.opencontainers.image.ref.name": "tianon/test@sha256:347290ddd775c1b85a3e381b09edde95242478eb65153e9b17225356f4c072ac" + } } - }, - { - "schemaVersion": 2, - "mediaType": "application/vnd.oci.image.index.v1+json", - "manifests": [ - { - "mediaType": "application/vnd.oci.image.manifest.v1+json", - "digest": "sha256:e2fc4e5012d16e7fe466f5291c476431beaa1f9b90a5c2125b493ed28e2aba57", - "size": 861, - "annotations": { - "com.docker.official-images.bashbrew.arch": "amd64", - "org.opencontainers.image.ref.name": "tianon/test@sha256:e2fc4e5012d16e7fe466f5291c476431beaa1f9b90a5c2125b493ed28e2aba57", - "org.opencontainers.image.revision": "3fb6ebca4163bf5b9cc496ac3e8f11cb1e754aee", - "org.opencontainers.image.source": "https://github.com/docker-library/hello-world.git#3fb6ebca4163bf5b9cc496ac3e8f11cb1e754aee:amd64/hello-world", - "org.opencontainers.image.url": "https://hub.docker.com/_/hello-world", - "org.opencontainers.image.version": "linux" + ], + [ + [ + "--type manifest", + "tianon/test@sha256:347290ddd775c1b85a3e381b09edde95242478eb65153e9b17225356f4c072ac" + ], + { + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.index.v1+json", + "manifests": [ + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:e2fc4e5012d16e7fe466f5291c476431beaa1f9b90a5c2125b493ed28e2aba57", + "size": 861, + "annotations": { + "com.docker.official-images.bashbrew.arch": "amd64", + "org.opencontainers.image.ref.name": "hello-world@sha256:e2fc4e5012d16e7fe466f5291c476431beaa1f9b90a5c2125b493ed28e2aba57", + "org.opencontainers.image.revision": "3fb6ebca4163bf5b9cc496ac3e8f11cb1e754aee", + "org.opencontainers.image.source": "https://github.com/docker-library/hello-world.git#3fb6ebca4163bf5b9cc496ac3e8f11cb1e754aee:amd64/hello-world", + "org.opencontainers.image.url": "https://hub.docker.com/_/hello-world", + "org.opencontainers.image.version": "linux" + } }, - "platform": { - "architecture": "amd64", - "os": "linux" + { + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "digest": "sha256:2f19ce27632e6baf4ebb1b582960d68948e52902c8cfac10133da0058f1dab23", + "size": 946, + "annotations": { + "com.docker.official-images.bashbrew.arch": "windows-amd64", + "org.opencontainers.image.ref.name": "hello-world@sha256:2f19ce27632e6baf4ebb1b582960d68948e52902c8cfac10133da0058f1dab23" + } + }, + { + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "digest": "sha256:3a0bd0fb5ad6dd6528dc78726b3df78e980b39b379e99c5a508904ec17cfafe5", + "size": 946, + "annotations": { + "com.docker.official-images.bashbrew.arch": "windows-amd64", + "org.opencontainers.image.ref.name": "hello-world@sha256:3a0bd0fb5ad6dd6528dc78726b3df78e980b39b379e99c5a508904ec17cfafe5" + } } + ] + } + ], + [ + [ + "--type blob", + "tianon/test@sha256:d2c94e258dcb3c5ac2798d32e1249e42ef01cba4841c2234249495f87264ac5a" + ], + "eyJhcmNoaXRlY3R1cmUiOiJhbWQ2NCIsImNvbmZpZyI6eyJFbnYiOlsiUEFUSD0vdXNyL2xvY2FsL3NiaW46L3Vzci9sb2NhbC9iaW46L3Vzci9zYmluOi91c3IvYmluOi9zYmluOi9iaW4iXSwiQ21kIjpbIi9oZWxsbyJdLCJXb3JraW5nRGlyIjoiLyIsIkFyZ3NFc2NhcGVkIjp0cnVlLCJPbkJ1aWxkIjpudWxsfSwiY3JlYXRlZCI6IjIwMjMtMDUtMDJUMTY6NDk6MjdaIiwiaGlzdG9yeSI6W3siY3JlYXRlZCI6IjIwMjMtMDUtMDJUMTY6NDk6MjdaIiwiY3JlYXRlZF9ieSI6IkNPUFkgaGVsbG8gLyAjIGJ1aWxka2l0IiwiY29tbWVudCI6ImJ1aWxka2l0LmRvY2tlcmZpbGUudjAifSx7ImNyZWF0ZWQiOiIyMDIzLTA1LTAyVDE2OjQ5OjI3WiIsImNyZWF0ZWRfYnkiOiJDTUQgW1wiL2hlbGxvXCJdIiwiY29tbWVudCI6ImJ1aWxka2l0LmRvY2tlcmZpbGUudjAiLCJlbXB0eV9sYXllciI6dHJ1ZX1dLCJvcyI6ImxpbnV4Iiwicm9vdGZzIjp7InR5cGUiOiJsYXllcnMiLCJkaWZmX2lkcyI6WyJzaGEyNTY6YWMyODgwMGVjOGJiMzhkNWMzNWI0OWQ0NWE2YWM0Nzc3NTQ0OTQxMTk5MDc1ZGZmOGM0ZWI2M2UwOTNhYTgxZSJdfX0=" + ], + [ + [ + "--head", + "--type manifest", + "tianon/test@sha256:347290ddd775c1b85a3e381b09edde95242478eb65153e9b17225356f4c072ac" + ], + { + "mediaType": "application/vnd.oci.image.index.v1+json", + "digest": "sha256:347290ddd775c1b85a3e381b09edde95242478eb65153e9b17225356f4c072ac", + "size": 1649, + "data": "ewoJInNjaGVtYVZlcnNpb24iOiAyLAoJIm1lZGlhVHlwZSI6ICJhcHBsaWNhdGlvbi92bmQub2NpLmltYWdlLmluZGV4LnYxK2pzb24iLAoJIm1hbmlmZXN0cyI6IFsKCQl7CgkJCSJtZWRpYVR5cGUiOiAiYXBwbGljYXRpb24vdm5kLm9jaS5pbWFnZS5tYW5pZmVzdC52MStqc29uIiwKCQkJImRpZ2VzdCI6ICJzaGEyNTY6ZTJmYzRlNTAxMmQxNmU3ZmU0NjZmNTI5MWM0NzY0MzFiZWFhMWY5YjkwYTVjMjEyNWI0OTNlZDI4ZTJhYmE1NyIsCgkJCSJzaXplIjogODYxLAoJCQkiYW5ub3RhdGlvbnMiOiB7CgkJCQkiY29tLmRvY2tlci5vZmZpY2lhbC1pbWFnZXMuYmFzaGJyZXcuYXJjaCI6ICJhbWQ2NCIsCgkJCQkib3JnLm9wZW5jb250YWluZXJzLmltYWdlLnJlZi5uYW1lIjogImhlbGxvLXdvcmxkQHNoYTI1NjplMmZjNGU1MDEyZDE2ZTdmZTQ2NmY1MjkxYzQ3NjQzMWJlYWExZjliOTBhNWMyMTI1YjQ5M2VkMjhlMmFiYTU3IiwKCQkJCSJvcmcub3BlbmNvbnRhaW5lcnMuaW1hZ2UucmV2aXNpb24iOiAiM2ZiNmViY2E0MTYzYmY1YjljYzQ5NmFjM2U4ZjExY2IxZTc1NGFlZSIsCgkJCQkib3JnLm9wZW5jb250YWluZXJzLmltYWdlLnNvdXJjZSI6ICJodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyLWxpYnJhcnkvaGVsbG8td29ybGQuZ2l0IzNmYjZlYmNhNDE2M2JmNWI5Y2M0OTZhYzNlOGYxMWNiMWU3NTRhZWU6YW1kNjQvaGVsbG8td29ybGQiLAoJCQkJIm9yZy5vcGVuY29udGFpbmVycy5pbWFnZS51cmwiOiAiaHR0cHM6Ly9odWIuZG9ja2VyLmNvbS9fL2hlbGxvLXdvcmxkIiwKCQkJCSJvcmcub3BlbmNvbnRhaW5lcnMuaW1hZ2UudmVyc2lvbiI6ICJsaW51eCIKCQkJfQoJCX0sCgkJewoJCQkibWVkaWFUeXBlIjogImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuZGlzdHJpYnV0aW9uLm1hbmlmZXN0LnYyK2pzb24iLAoJCQkiZGlnZXN0IjogInNoYTI1NjoyZjE5Y2UyNzYzMmU2YmFmNGViYjFiNTgyOTYwZDY4OTQ4ZTUyOTAyYzhjZmFjMTAxMzNkYTAwNThmMWRhYjIzIiwKCQkJInNpemUiOiA5NDYsCgkJCSJhbm5vdGF0aW9ucyI6IHsKCQkJCSJjb20uZG9ja2VyLm9mZmljaWFsLWltYWdlcy5iYXNoYnJldy5hcmNoIjogIndpbmRvd3MtYW1kNjQiLAoJCQkJIm9yZy5vcGVuY29udGFpbmVycy5pbWFnZS5yZWYubmFtZSI6ICJoZWxsby13b3JsZEBzaGEyNTY6MmYxOWNlMjc2MzJlNmJhZjRlYmIxYjU4Mjk2MGQ2ODk0OGU1MjkwMmM4Y2ZhYzEwMTMzZGEwMDU4ZjFkYWIyMyIKCQkJfQoJCX0sCgkJewoJCQkibWVkaWFUeXBlIjogImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuZGlzdHJpYnV0aW9uLm1hbmlmZXN0LnYyK2pzb24iLAoJCQkiZGlnZXN0IjogInNoYTI1NjozYTBiZDBmYjVhZDZkZDY1MjhkYzc4NzI2YjNkZjc4ZTk4MGIzOWIzNzllOTljNWE1MDg5MDRlYzE3Y2ZhZmU1IiwKCQkJInNpemUiOiA5NDYsCgkJCSJhbm5vdGF0aW9ucyI6IHsKCQkJCSJjb20uZG9ja2VyLm9mZmljaWFsLWltYWdlcy5iYXNoYnJldy5hcmNoIjogIndpbmRvd3MtYW1kNjQiLAoJCQkJIm9yZy5vcGVuY29udGFpbmVycy5pbWFnZS5yZWYubmFtZSI6ICJoZWxsby13b3JsZEBzaGEyNTY6M2EwYmQwZmI1YWQ2ZGQ2NTI4ZGM3ODcyNmIzZGY3OGU5ODBiMzliMzc5ZTk5YzVhNTA4OTA0ZWMxN2NmYWZlNSIKCQkJfQoJCX0KCV0KfQo=" + } + ], + [ + [ + "--head", + "--type blob", + "tianon/test@sha256:d2c94e258dcb3c5ac2798d32e1249e42ef01cba4841c2234249495f87264ac5a" + ], + { + "mediaType": "application/octet-stream", + "digest": "sha256:d2c94e258dcb3c5ac2798d32e1249e42ef01cba4841c2234249495f87264ac5a", + "size": 581, + "data": "eyJhcmNoaXRlY3R1cmUiOiJhbWQ2NCIsImNvbmZpZyI6eyJFbnYiOlsiUEFUSD0vdXNyL2xvY2FsL3NiaW46L3Vzci9sb2NhbC9iaW46L3Vzci9zYmluOi91c3IvYmluOi9zYmluOi9iaW4iXSwiQ21kIjpbIi9oZWxsbyJdLCJXb3JraW5nRGlyIjoiLyIsIkFyZ3NFc2NhcGVkIjp0cnVlLCJPbkJ1aWxkIjpudWxsfSwiY3JlYXRlZCI6IjIwMjMtMDUtMDJUMTY6NDk6MjdaIiwiaGlzdG9yeSI6W3siY3JlYXRlZCI6IjIwMjMtMDUtMDJUMTY6NDk6MjdaIiwiY3JlYXRlZF9ieSI6IkNPUFkgaGVsbG8gLyAjIGJ1aWxka2l0IiwiY29tbWVudCI6ImJ1aWxka2l0LmRvY2tlcmZpbGUudjAifSx7ImNyZWF0ZWQiOiIyMDIzLTA1LTAyVDE2OjQ5OjI3WiIsImNyZWF0ZWRfYnkiOiJDTUQgW1wiL2hlbGxvXCJdIiwiY29tbWVudCI6ImJ1aWxka2l0LmRvY2tlcmZpbGUudjAiLCJlbXB0eV9sYXllciI6dHJ1ZX1dLCJvcyI6ImxpbnV4Iiwicm9vdGZzIjp7InR5cGUiOiJsYXllcnMiLCJkaWZmX2lkcyI6WyJzaGEyNTY6YWMyODgwMGVjOGJiMzhkNWMzNWI0OWQ0NWE2YWM0Nzc3NTQ0OTQxMTk5MDc1ZGZmOGM0ZWI2M2UwOTNhYTgxZSJdfX0=" + } + ], + [ + [ + "--head", + "--type blob", + "tianon/true@sha256:25be82253336f0b8c4347bc4ecbbcdc85d0e0f118ccf8dc2e119c0a47a0a486e" + ], + { + "mediaType": "application/octet-stream", + "digest": "sha256:25be82253336f0b8c4347bc4ecbbcdc85d0e0f118ccf8dc2e119c0a47a0a486e", + "size": 396 + } + ], + [ + [ + "--head", + "--type manifest", + "tianon/true:oci@sha256:9ef42f1d602fb423fad935aac1caa0cfdbce1ad7edce64d080a4eb7b13f7cd9d" + ], + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:9ef42f1d602fb423fad935aac1caa0cfdbce1ad7edce64d080a4eb7b13f7cd9d", + "size": 1165 + } + ], + [ + [ + "--type blob", + "tianon/true@sha256:25be82253336f0b8c4347bc4ecbbcdc85d0e0f118ccf8dc2e119c0a47a0a486e" + ], + "ewoJImFyY2hpdGVjdHVyZSI6ICJhbWQ2NCIsCgkiY29uZmlnIjogewoJCSJDbWQiOiBbCgkJCSIvdHJ1ZSIKCQldCgl9LAoJImNyZWF0ZWQiOiAiMjAyMy0wMi0wMVQwNjo1MToxMVoiLAoJImhpc3RvcnkiOiBbCgkJewoJCQkiY3JlYXRlZCI6ICIyMDIzLTAyLTAxVDA2OjUxOjExWiIsCgkJCSJjcmVhdGVkX2J5IjogImh0dHBzOi8vZ2l0aHViLmNvbS90aWFub24vZG9ja2VyZmlsZXMvdHJlZS9tYXN0ZXIvdHJ1ZSIKCQl9CgldLAoJIm9zIjogImxpbnV4IiwKCSJyb290ZnMiOiB7CgkJImRpZmZfaWRzIjogWwoJCQkic2hhMjU2OjY1YjVhNDU5M2NjNjFkM2VhNmQzNTVmYjk3YzA0MzBkODIwZWUyMWFhODUzNWY1ZGU0NWU3NWMzMTk1NGI3NDMiCgkJXSwKCQkidHlwZSI6ICJsYXllcnMiCgl9Cn0K" + ], + [ + [ + "--type manifest", + "tianon/true:oci@sha256:9ef42f1d602fb423fad935aac1caa0cfdbce1ad7edce64d080a4eb7b13f7cd9d" + ], + { + "config": { + "data": "ewoJImFyY2hpdGVjdHVyZSI6ICJhbWQ2NCIsCgkiY29uZmlnIjogewoJCSJDbWQiOiBbCgkJCSIvdHJ1ZSIKCQldCgl9LAoJImNyZWF0ZWQiOiAiMjAyMy0wMi0wMVQwNjo1MToxMVoiLAoJImhpc3RvcnkiOiBbCgkJewoJCQkiY3JlYXRlZCI6ICIyMDIzLTAyLTAxVDA2OjUxOjExWiIsCgkJCSJjcmVhdGVkX2J5IjogImh0dHBzOi8vZ2l0aHViLmNvbS90aWFub24vZG9ja2VyZmlsZXMvdHJlZS9tYXN0ZXIvdHJ1ZSIKCQl9CgldLAoJIm9zIjogImxpbnV4IiwKCSJyb290ZnMiOiB7CgkJImRpZmZfaWRzIjogWwoJCQkic2hhMjU2OjY1YjVhNDU5M2NjNjFkM2VhNmQzNTVmYjk3YzA0MzBkODIwZWUyMWFhODUzNWY1ZGU0NWU3NWMzMTk1NGI3NDMiCgkJXSwKCQkidHlwZSI6ICJsYXllcnMiCgl9Cn0K", + "digest": "sha256:25be82253336f0b8c4347bc4ecbbcdc85d0e0f118ccf8dc2e119c0a47a0a486e", + "mediaType": "application/vnd.oci.image.config.v1+json", + "size": 396 }, - { - "mediaType": "application/vnd.docker.distribution.manifest.v2+json", - "digest": "sha256:2f19ce27632e6baf4ebb1b582960d68948e52902c8cfac10133da0058f1dab23", - "size": 946, - "annotations": { - "com.docker.official-images.bashbrew.arch": "windows-amd64", - "org.opencontainers.image.ref.name": "tianon/test@sha256:2f19ce27632e6baf4ebb1b582960d68948e52902c8cfac10133da0058f1dab23" + "layers": [ + { + "data": "H4sIAAAAAAACAyspKk1loDEwAAJTU1MwDQTotIGhuQmcDRE3MzM0YlAwYKADKC0uSSxSUGAYoaDe1ceNiZERzmdisGMA8SoYHMB8Byx6HBgsGGA6QDQrmiwyXQPl1cDlIUG9wYaflWEUDDgAAIAGdJIABAAA", + "digest": "sha256:1c51fc286aa95d9413226599576bafa38490b1e292375c90de095855b64caea6", + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "size": 117 + } + ], + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "schemaVersion": 2 + } + ], + [ + [ + "tianon/true:oci@sha256:9ef42f1d602fb423fad935aac1caa0cfdbce1ad7edce64d080a4eb7b13f7cd9d" + ], + { + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.index.v1+json", + "manifests": [ + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:9ef42f1d602fb423fad935aac1caa0cfdbce1ad7edce64d080a4eb7b13f7cd9d", + "size": 1165, + "annotations": { + "com.docker.official-images.bashbrew.arch": "amd64", + "org.opencontainers.image.ref.name": "tianon/true:oci@sha256:9ef42f1d602fb423fad935aac1caa0cfdbce1ad7edce64d080a4eb7b13f7cd9d" + }, + "platform": { + "architecture": "amd64", + "os": "linux" + } + } + ], + "annotations": { + "org.opencontainers.image.ref.name": "tianon/true:oci@sha256:9ef42f1d602fb423fad935aac1caa0cfdbce1ad7edce64d080a4eb7b13f7cd9d" + } + } + ], + [ + [ + "--head", + "oisupport/staging-amd64:71756dd75e41c4bc5144b64d36b4834a5a960c495470915eb69f96e9f2cb6694" + ], + { + "mediaType": "application/vnd.oci.image.index.v1+json", + "digest": "sha256:09dd1c0183f992a4507d6e562a8e079b8583d19aaf8d991b0d22711c6b4525d7", + "size": 1265 + } + ], + [ + [ + "--head", + "oisupport/staging-amd64:71756dd75e41c4bc5144b64d36b4834a5a960c495470915eb69f96e9f2cb6694" + ], + { + "mediaType": "application/vnd.oci.image.index.v1+json", + "digest": "sha256:09dd1c0183f992a4507d6e562a8e079b8583d19aaf8d991b0d22711c6b4525d7", + "size": 1265 + } + ], + [ + [ + "--type manifest", + "oisupport/staging-amd64:71756dd75e41c4bc5144b64d36b4834a5a960c495470915eb69f96e9f2cb6694" + ], + { + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.index.v1+json", + "manifests": [ + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:4c3d07b2fed560ab0012452aa8a6f58533ddf2d4a3845fa89b74d9455816b454", + "size": 1998, + "annotations": { + "org.opencontainers.image.revision": "77b9b7833f8dd6be07104b214193788795a320ff", + "org.opencontainers.image.source": "https://github.com/docker/notary-official-images.git#77b9b7833f8dd6be07104b214193788795a320ff:notary-server", + "org.opencontainers.image.url": "https://hub.docker.com/_/notary", + "org.opencontainers.image.version": "server-0.7.0" + }, + "platform": { + "architecture": "amd64", + "os": "linux" + } }, - "platform": { - "architecture": "amd64", - "os": "windows", - "os.version": "10.0.20348.2340" + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:692819af7e57efe94abadb451e05aa5eb042a540a2eae7095d37507dbd66dc94", + "size": 839, + "annotations": { + "vnd.docker.reference.digest": "sha256:4c3d07b2fed560ab0012452aa8a6f58533ddf2d4a3845fa89b74d9455816b454", + "vnd.docker.reference.type": "attestation-manifest" + }, + "platform": { + "architecture": "unknown", + "os": "unknown" + } } - }, - { - "mediaType": "application/vnd.docker.distribution.manifest.v2+json", - "digest": "sha256:3a0bd0fb5ad6dd6528dc78726b3df78e980b39b379e99c5a508904ec17cfafe5", - "size": 946, - "annotations": { - "com.docker.official-images.bashbrew.arch": "windows-amd64", - "org.opencontainers.image.ref.name": "tianon/test@sha256:3a0bd0fb5ad6dd6528dc78726b3df78e980b39b379e99c5a508904ec17cfafe5" + ] + } + ], + [ + [ + "oisupport/staging-amd64:71756dd75e41c4bc5144b64d36b4834a5a960c495470915eb69f96e9f2cb6694" + ], + { + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.index.v1+json", + "manifests": [ + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:4c3d07b2fed560ab0012452aa8a6f58533ddf2d4a3845fa89b74d9455816b454", + "size": 1998, + "annotations": { + "com.docker.official-images.bashbrew.arch": "amd64", + "org.opencontainers.image.ref.name": "oisupport/staging-amd64:71756dd75e41c4bc5144b64d36b4834a5a960c495470915eb69f96e9f2cb6694@sha256:4c3d07b2fed560ab0012452aa8a6f58533ddf2d4a3845fa89b74d9455816b454", + "org.opencontainers.image.revision": "77b9b7833f8dd6be07104b214193788795a320ff", + "org.opencontainers.image.source": "https://github.com/docker/notary-official-images.git#77b9b7833f8dd6be07104b214193788795a320ff:notary-server", + "org.opencontainers.image.url": "https://hub.docker.com/_/notary", + "org.opencontainers.image.version": "server-0.7.0" + }, + "platform": { + "architecture": "amd64", + "os": "linux" + } }, - "platform": { - "architecture": "amd64", - "os": "windows", - "os.version": "10.0.17763.5576" + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:692819af7e57efe94abadb451e05aa5eb042a540a2eae7095d37507dbd66dc94", + "size": 839, + "annotations": { + "com.docker.official-images.bashbrew.arch": "amd64", + "org.opencontainers.image.ref.name": "oisupport/staging-amd64:71756dd75e41c4bc5144b64d36b4834a5a960c495470915eb69f96e9f2cb6694@sha256:692819af7e57efe94abadb451e05aa5eb042a540a2eae7095d37507dbd66dc94", + "vnd.docker.reference.digest": "sha256:4c3d07b2fed560ab0012452aa8a6f58533ddf2d4a3845fa89b74d9455816b454", + "vnd.docker.reference.type": "attestation-manifest" + }, + "platform": { + "architecture": "unknown", + "os": "unknown" + } } + ], + "annotations": { + "org.opencontainers.image.ref.name": "oisupport/staging-amd64:71756dd75e41c4bc5144b64d36b4834a5a960c495470915eb69f96e9f2cb6694@sha256:09dd1c0183f992a4507d6e562a8e079b8583d19aaf8d991b0d22711c6b4525d7" } - ], - "annotations": { - "org.opencontainers.image.ref.name": "tianon/test@sha256:347290ddd775c1b85a3e381b09edde95242478eb65153e9b17225356f4c072ac" } - } + ], + [ + [ + "tianon/this-is-a-repository-that-will-never-ever-exist-$RANDOM-$RANDOM:$RANDOM-$RANDOM" + ], + null + ], + [ + [ + "--head", + "tianon/this-is-a-repository-that-will-never-ever-exist-$RANDOM-$RANDOM:$RANDOM-$RANDOM" + ], + null + ], + [ + [ + "tianon/test@sha256:0000000000000000000000000000000000000000000000000000000000000000" + ], + null + ] ] diff --git a/.test/test.sh b/.test/test.sh index adb9900..a942e32 100755 --- a/.test/test.sh +++ b/.test/test.sh @@ -48,8 +48,50 @@ lookup=( # tianon/test:index-no-platform-smaller - a "broken" index with *zero* platform objects in it (so every manifest requires a platform lookup) 'tianon/test@sha256:347290ddd775c1b85a3e381b09edde95242478eb65153e9b17225356f4c072ac' # (doing these in the same run means the manifest from above should be cached and exercise more codepaths for better coverage) + + --type manifest 'tianon/test@sha256:347290ddd775c1b85a3e381b09edde95242478eb65153e9b17225356f4c072ac' # same manifest again, but without SynthesizeIndex + --type blob 'tianon/test@sha256:d2c94e258dcb3c5ac2798d32e1249e42ef01cba4841c2234249495f87264ac5a' # first config blob from the above + # and again, but this time HEADs + --head --type manifest 'tianon/test@sha256:347290ddd775c1b85a3e381b09edde95242478eb65153e9b17225356f4c072ac' + --head --type blob 'tianon/test@sha256:d2c94e258dcb3c5ac2798d32e1249e42ef01cba4841c2234249495f87264ac5a' + + # again with things that aren't cached yet (tianon/true:oci, specifically) + --head --type blob 'tianon/true@sha256:25be82253336f0b8c4347bc4ecbbcdc85d0e0f118ccf8dc2e119c0a47a0a486e' # config blob + --head --type manifest 'tianon/true:oci@sha256:9ef42f1d602fb423fad935aac1caa0cfdbce1ad7edce64d080a4eb7b13f7cd9d' + --type blob 'tianon/true@sha256:25be82253336f0b8c4347bc4ecbbcdc85d0e0f118ccf8dc2e119c0a47a0a486e' # config blob + --type manifest 'tianon/true:oci@sha256:9ef42f1d602fb423fad935aac1caa0cfdbce1ad7edce64d080a4eb7b13f7cd9d' + 'tianon/true:oci@sha256:9ef42f1d602fb423fad935aac1caa0cfdbce1ad7edce64d080a4eb7b13f7cd9d' + + # tag lookup! (but with a hopefully stable example tag -- a build of notary:server) + --head 'oisupport/staging-amd64:71756dd75e41c4bc5144b64d36b4834a5a960c495470915eb69f96e9f2cb6694' + --head 'oisupport/staging-amd64:71756dd75e41c4bc5144b64d36b4834a5a960c495470915eb69f96e9f2cb6694' # twice, to exercise "tag is cached" case + --type manifest 'oisupport/staging-amd64:71756dd75e41c4bc5144b64d36b4834a5a960c495470915eb69f96e9f2cb6694' + 'oisupport/staging-amd64:71756dd75e41c4bc5144b64d36b4834a5a960c495470915eb69f96e9f2cb6694' + + # exercise 404 codepaths + "tianon/this-is-a-repository-that-will-never-ever-exist-$RANDOM-$RANDOM:$RANDOM-$RANDOM" + --head "tianon/this-is-a-repository-that-will-never-ever-exist-$RANDOM-$RANDOM:$RANDOM-$RANDOM" + 'tianon/test@sha256:0000000000000000000000000000000000000000000000000000000000000000' ) -"$dir/../bin/lookup" "${lookup[@]}" | jq -s > "$dir/lookup-test.json" +"$dir/../bin/lookup" "${lookup[@]}" | jq -s ' + [ + reduce ( + $ARGS.positional[] + | if startswith("tianon/this-is-a-repository-that-will-never-ever-exist-") then + gsub("[0-9]+"; "$RANDOM") + else . end + ) as $a ([]; + if .[-1][-1] == "--type" then + .[-1][-1] += " " + $a + elif length > 0 and (.[-1][-1] | startswith("--")) then + .[-1] += [$a] + else + . += [[$a]] + end + ), + . + ] | transpose +' --args -- "${lookup[@]}" > "$dir/lookup-test.json" # don't leave around the "-cover" versions of these binaries rm -f "$dir/../bin/builds" "$dir/../bin/lookup" diff --git a/cmd/lookup/main.go b/cmd/lookup/main.go index 0d47f30..76fd208 100644 --- a/cmd/lookup/main.go +++ b/cmd/lookup/main.go @@ -5,6 +5,7 @@ package main import ( "context" "encoding/json" + "io" "os" "os/signal" @@ -15,21 +16,79 @@ func main() { ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) defer stop() - for _, img := range os.Args[1:] { + var ( + zeroOpts registry.LookupOptions + opts = zeroOpts + ) + + args := os.Args[1:] + for len(args) > 0 { + img := args[0] + args = args[1:] + switch img { + case "--type": + opts.Type = registry.LookupType(args[0]) + args = args[1:] + continue + case "--head": + opts.Head = true + continue + } + ref, err := registry.ParseRef(img) if err != nil { panic(err) } - index, err := registry.SynthesizeIndex(ctx, ref) - if err != nil { - panic(err) + var obj any + if opts == zeroOpts { + // if we have no explicit type and didn't request a HEAD, invoke SynthesizeIndex instead of Lookup + obj, err = registry.SynthesizeIndex(ctx, ref) + if err != nil { + panic(err) + } + } else { + r, err := registry.Lookup(ctx, ref, &opts) + if err != nil { + panic(err) + } + if r != nil { + desc := r.Descriptor() + if opts.Head { + obj = desc + } else { + b, err := io.ReadAll(r) + if err != nil { + r.Close() + panic(err) + } + if opts.Type == registry.LookupTypeManifest { + // if it was a manifest lookup, cast the byte slice to json.RawMessage so we get the actual JSON (not base64) + obj = json.RawMessage(b) + } else { + obj = b + } + } + err = r.Close() + if err != nil { + panic(err) + } + } else { + obj = nil + } } e := json.NewEncoder(os.Stdout) e.SetIndent("", "\t") - if err := e.Encode(index); err != nil { + if err := e.Encode(obj); err != nil { panic(err) } + + // reset state + opts = zeroOpts + } + + if opts != zeroOpts { + panic("dangling --type, --head, etc (without a following reference for it to apply to)") } } diff --git a/go.mod b/go.mod index f900ad9..d8b6b27 100644 --- a/go.mod +++ b/go.mod @@ -20,5 +20,5 @@ require ( google.golang.org/protobuf v1.28.1 // indirect ) -// https://github.com/cue-labs/oci/pull/27 -replace cuelabs.dev/go/oci/ociregistry => github.com/tianon/cuelabs-oci/ociregistry v0.0.0-20240216044210-8aa0c990bd77 +// https://github.com/cue-labs/oci/pull/29 +replace cuelabs.dev/go/oci/ociregistry => github.com/tianon/cuelabs-oci/ociregistry v0.0.0-20240322151419-7d3242933116 diff --git a/go.sum b/go.sum index d3d7a29..0ab5728 100644 --- a/go.sum +++ b/go.sum @@ -32,8 +32,8 @@ github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVs github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/tianon/cuelabs-oci/ociregistry v0.0.0-20240216044210-8aa0c990bd77 h1:9EPZm+sGlYHo6LleMXWR6s3P8SJEYA7/aovpJ76JSpw= -github.com/tianon/cuelabs-oci/ociregistry v0.0.0-20240216044210-8aa0c990bd77/go.mod h1:ApHceQLLwcOkCEXM1+DyCXTHEJhNGDpJ2kmV6axsx24= +github.com/tianon/cuelabs-oci/ociregistry v0.0.0-20240322151419-7d3242933116 h1:ZDy4uRAhzODJXRo4EoNpJTCiSeOs8wwrkfMJy3JyDps= +github.com/tianon/cuelabs-oci/ociregistry v0.0.0-20240322151419-7d3242933116/go.mod h1:pK23AUVXuNzzTpfMCA06sxZGeVQ/75FdVtW249de9Uo= golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/registry/cache.go b/registry/cache.go index 3346d84..2d913a3 100644 --- a/registry/cache.go +++ b/registry/cache.go @@ -21,8 +21,7 @@ func RegistryCache(r ociregistry.Interface) ociregistry.Interface { registry: r, // TODO support "nil" here so this can be a poor-man's ocimem implementation? 👀 see also https://github.com/cue-labs/oci/issues/24 has: map[string]bool{}, tags: map[string]ociregistry.Digest{}, - types: map[ociregistry.Digest]string{}, - data: map[ociregistry.Digest][]byte{}, + data: map[ociregistry.Digest]ociregistry.Descriptor{}, } } @@ -32,11 +31,10 @@ type registryCache struct { registry ociregistry.Interface // https://github.com/cue-labs/oci/issues/24 - mu sync.Mutex // TODO some kind of per-object/name/digest mutex so we don't request the same object from the upstream registry concurrently (on *top* of our maps mutex)? - has map[string]bool // "repo/name@digest" => true (whether a given repo has the given digest) - tags map[string]ociregistry.Digest // "repo/name:tag" => digest - types map[ociregistry.Digest]string // digest => "mediaType" (most recent *storing* / "cache-miss" lookup wins, in the case of upstream/cross-repo ambiguity) - data map[ociregistry.Digest][]byte // digest => data + mu sync.Mutex // TODO some kind of per-object/name/digest mutex so we don't request the same object from the upstream registry concurrently (on *top* of our maps mutex)? + has map[string]bool // "repo/name@digest" => true (whether a given repo has the given digest) + tags map[string]ociregistry.Digest // "repo/name:tag" => digest + data map[ociregistry.Digest]ociregistry.Descriptor // digest => mediaType+size(+data) (most recent *storing* / "cache-miss" lookup wins, in the case of upstream/cross-repo ambiguity) } func cacheKeyDigest(repo string, digest ociregistry.Digest) string { @@ -52,41 +50,38 @@ func (rc *registryCache) getBlob(ctx context.Context, repo string, digest ocireg rc.mu.Lock() defer rc.mu.Unlock() - if b, ok := rc.data[digest]; ok && rc.has[cacheKeyDigest(repo, digest)] { - return ocimem.NewBytesReader(b, ociregistry.Descriptor{ - MediaType: rc.types[digest], - Digest: digest, - Size: int64(len(b)), - }), nil + if desc, ok := rc.data[digest]; ok && desc.Data != nil && rc.has[cacheKeyDigest(repo, digest)] { + return ocimem.NewBytesReader(desc.Data, desc), nil } r, err := f(ctx, repo, digest) if err != nil { return nil, err } - //defer r.Close() + // defer r.Close() happens later when we know we aren't making Close the caller's responsibility desc := r.Descriptor() + digest = desc.Digest // if this isn't a no-op, we've got a naughty registry - rc.has[cacheKeyDigest(repo, desc.Digest)] = true - rc.types[desc.Digest] = desc.MediaType + rc.has[cacheKeyDigest(repo, digest)] = true + + if desc.Size > manifestSizeLimit { + rc.data[digest] = desc + return r, nil + } + defer r.Close() - b, err := io.ReadAll(r) + desc.Data, err = io.ReadAll(r) if err != nil { - r.Close() return nil, err } if err := r.Close(); err != nil { return nil, err } - if len(b) <= manifestSizeLimit { - rc.data[desc.Digest] = b - } else { - delete(rc.data, desc.Digest) - } + rc.data[digest] = desc - return ocimem.NewBytesReader(b, desc), nil + return ocimem.NewBytesReader(desc.Data, desc), nil } func (rc *registryCache) GetBlob(ctx context.Context, repo string, digest ociregistry.Digest) (ociregistry.BlobReader, error) { @@ -104,12 +99,8 @@ func (rc *registryCache) GetTag(ctx context.Context, repo string, tag string) (o tagKey := cacheKeyTag(repo, tag) if digest, ok := rc.tags[tagKey]; ok { - if b, ok := rc.data[digest]; ok { - return ocimem.NewBytesReader(b, ociregistry.Descriptor{ - MediaType: rc.types[digest], - Digest: digest, - Size: int64(len(b)), - }), nil + if desc, ok := rc.data[digest]; ok && desc.Data != nil { + return ocimem.NewBytesReader(desc.Data, desc), nil } } @@ -117,30 +108,99 @@ func (rc *registryCache) GetTag(ctx context.Context, repo string, tag string) (o if err != nil { return nil, err } - //defer r.Close() + // defer r.Close() happens later when we know we aren't making Close the caller's responsibility desc := r.Descriptor() rc.has[cacheKeyDigest(repo, desc.Digest)] = true rc.tags[tagKey] = desc.Digest - rc.types[desc.Digest] = desc.MediaType - b, err := io.ReadAll(r) + if desc.Size > manifestSizeLimit { + rc.data[desc.Digest] = desc + return r, nil + } + defer r.Close() + + desc.Data, err = io.ReadAll(r) if err != nil { - r.Close() return nil, err } if err := r.Close(); err != nil { return nil, err } - if len(b) <= manifestSizeLimit { - rc.data[desc.Digest] = b - } else { - delete(rc.data, desc.Digest) + rc.data[desc.Digest] = desc + + return ocimem.NewBytesReader(desc.Data, desc), nil +} + +func (rc *registryCache) resolveBlob(ctx context.Context, repo string, digest ociregistry.Digest, f func(ctx context.Context, repo string, digest ociregistry.Digest) (ociregistry.Descriptor, error)) (ociregistry.Descriptor, error) { + rc.mu.Lock() + defer rc.mu.Unlock() + + if desc, ok := rc.data[digest]; ok && rc.has[cacheKeyDigest(repo, digest)] { + return desc, nil + } + + desc, err := f(ctx, repo, digest) + if err != nil { + return desc, err + } + + digest = desc.Digest // if this isn't a no-op, we've got a naughty registry + + rc.has[cacheKeyDigest(repo, digest)] = true + + // carefully copy only valid Resolve* fields such that any other existing fields are kept (this matters more if we ever make our mutexes better/less aggressive 👀) + if d, ok := rc.data[digest]; ok { + d.MediaType = desc.MediaType + d.Digest = desc.Digest + d.Size = desc.Size + desc = d + } + rc.data[digest] = desc + + return desc, nil +} + +func (rc *registryCache) ResolveManifest(ctx context.Context, repo string, digest ociregistry.Digest) (ociregistry.Descriptor, error) { + return rc.resolveBlob(ctx, repo, digest, rc.registry.ResolveManifest) +} + +func (rc *registryCache) ResolveBlob(ctx context.Context, repo string, digest ociregistry.Digest) (ociregistry.Descriptor, error) { + return rc.resolveBlob(ctx, repo, digest, rc.registry.ResolveBlob) +} + +func (rc *registryCache) ResolveTag(ctx context.Context, repo string, tag string) (ociregistry.Descriptor, error) { + rc.mu.Lock() + defer rc.mu.Unlock() + + tagKey := cacheKeyTag(repo, tag) + + if digest, ok := rc.tags[tagKey]; ok { + if desc, ok := rc.data[digest]; ok { + return desc, nil + } + } + + desc, err := rc.registry.ResolveTag(ctx, repo, tag) + if err != nil { + return desc, err + } + + rc.has[cacheKeyDigest(repo, desc.Digest)] = true + rc.tags[tagKey] = desc.Digest + + // carefully copy only valid Resolve* fields such that any other existing fields are kept (this matters more if we ever make our mutexes better/less aggressive 👀) + if d, ok := rc.data[desc.Digest]; ok { + d.MediaType = desc.MediaType + d.Digest = desc.Digest + d.Size = desc.Size + desc = d } + rc.data[desc.Digest] = desc - return ocimem.NewBytesReader(b, desc), nil + return desc, nil } // TODO more methods (currently only implements what's actually necessary for SynthesizeIndex) diff --git a/registry/client.go b/registry/client.go index a355a90..db0a999 100644 --- a/registry/client.go +++ b/registry/client.go @@ -25,18 +25,24 @@ func Client(host string, opts *ociclient.Options) (ociregistry.Interface, error) if opts != nil { clientOptions = *opts } - if clientOptions.HTTPClient == nil { - clientOptions.HTTPClient = http.DefaultClient + if clientOptions.Transport == nil { + clientOptions.Transport = http.DefaultTransport } // if we have a rate limiter configured for this registry, shim it in if limiter, ok := registryRateLimiters[host]; ok { - clientOptions.HTTPClient = &rateLimitedRetryingDoer{ - doer: clientOptions.HTTPClient, - limiter: limiter, + clientOptions.Transport = &rateLimitedRetryingRoundTripper{ + roundTripper: clientOptions.Transport, + limiter: limiter, } } + // install the "authorization" wrapper/shim + clientOptions.Transport = ociauth.NewStdTransport(ociauth.StdTransportParams{ + Config: authConfig, + Transport: clientOptions.Transport, + }) + connectHost := host if host == dockerHubCanonical { connectHost = dockerHubConnect @@ -46,14 +52,6 @@ func Client(host string, opts *ociclient.Options) (ociregistry.Interface, error) // TODO some way for callers to specify that their "localhost" *does* require TLS (maybe only do this if `opts == nil`, but then users cannot supply *any* options and still get help setting Insecure for localhost 🤔 -- at least this is a more narrow use case than the opposite of not having a way to have non-localhost insecure registries) } - if clientOptions.Authorizer == nil { - // TODO https://github.com/cue-labs/oci/pull/28 -- ideally we'd set this sooner, but https://github.com/cue-labs/oci/blob/5ebe80b0a9a67ae83802d1fb1a189a8f0d089fb0/ociregistry/ociclient/client.go#L278-L282 means we have to do it late in case we installed a rate limiting HTTPClient (or the caller provided a custom one) - clientOptions.Authorizer = ociauth.NewStdAuthorizer(ociauth.StdAuthorizerParams{ - Config: authConfig, - HTTPClient: clientOptions.HTTPClient, - }) - } - hostOptions := clientOptions // make a copy, since "ociclient.New" mutates it (such that sharing the object afterwards probably isn't the best idea -- they'll have the same DebugID if so, which isn't ideal) client, err := ociclient.New(connectHost, &hostOptions) if err != nil { @@ -86,7 +84,7 @@ func Client(host string, opts *ociclient.Options) (ociregistry.Interface, error) default: return nil, fmt.Errorf("unsupported DOCKERHUB_PUBLIC_PROXY (with path)") } - // TODO complain about other URL bits (unsupported by "ociclient" except via custom "HTTPClient" / "HTTPDoer") + // TODO complain about other URL bits (unsupported by "ociclient" except via custom "RoundTripper") } else if proxy := os.Getenv("DOCKERHUB_PUBLIC_PROXY_HOST"); proxy != "" { proxyHost = proxy } diff --git a/registry/lookup.go b/registry/lookup.go new file mode 100644 index 0000000..ec0ca01 --- /dev/null +++ b/registry/lookup.go @@ -0,0 +1,99 @@ +package registry + +import ( + "context" + "errors" + "fmt" + "strings" + + "cuelabs.dev/go/oci/ociregistry" + "cuelabs.dev/go/oci/ociregistry/ocimem" +) + +// see `LookupType*` consts for possible values for this type +type LookupType string + +const ( + LookupTypeManifest LookupType = "manifest" + LookupTypeBlob LookupType = "blob" +) + +type LookupOptions struct { + // unspecified implies [LookupTypeManifest] + Type LookupType + + // whether or not to do a HEAD instead of a GET (will still return an [ociregistry.BlobReader], but with an empty body / zero bytes) + Head bool +} + +// a wrapper around [ociregistry.Interface.GetManifest] (and `GetTag`, `GetBlob`, and the `Resolve*` versions of the above) that accepts a [Reference] and always returns a [ociregistry.BlobReader] (in the case of a HEAD request, it will be a zero-length reader with just a valid descriptor) +func Lookup(ctx context.Context, ref Reference, opts *LookupOptions) (ociregistry.BlobReader, error) { + client, err := Client(ref.Host, nil) + if err != nil { + return nil, fmt.Errorf("%s: failed getting client: %w", ref, err) + } + + var o LookupOptions + if opts != nil { + o = *opts + } + + var ( + r ociregistry.BlobReader + desc ociregistry.Descriptor + ) + switch o.Type { + case LookupTypeManifest, "": + if ref.Digest != "" { + if o.Head { + desc, err = client.ResolveManifest(ctx, ref.Repository, ref.Digest) + } else { + r, err = client.GetManifest(ctx, ref.Repository, ref.Digest) + } + } else { + tag := ref.Tag + if tag == "" { + tag = "latest" + } + if o.Head { + desc, err = client.ResolveTag(ctx, ref.Repository, tag) + } else { + r, err = client.GetTag(ctx, ref.Repository, tag) + } + } + + case LookupTypeBlob: + // TODO error if Digest == "" ? (ociclient already does for us, so we can probably just pass it through here without much worry) + if o.Head { + desc, err = client.ResolveBlob(ctx, ref.Repository, ref.Digest) + } else { + r, err = client.GetBlob(ctx, ref.Repository, ref.Digest) + } + + default: + return nil, fmt.Errorf("unknown LookupType: %q", o.Type) + } + + // normalize 404 and 404-like to nil return (so it's easier to detect) + if err != nil { + if errors.Is(err, ociregistry.ErrBlobUnknown) || + errors.Is(err, ociregistry.ErrManifestUnknown) || + errors.Is(err, ociregistry.ErrNameUnknown) { + // obvious 404 cases + return nil, nil + } + // https://github.com/cue-labs/oci/issues/26 + if errStr := strings.TrimPrefix(err.Error(), "error response: "); strings.HasPrefix(errStr, "404 ") || + // 401 often means "repository not found" (due to the nature of public/private mixing on Hub and the fact that ociauth definitely handled any possible authentication for us, so if we're still getting 401 it's unavoidable and might as well be 404) + strings.HasPrefix(errStr, "401 ") { + return nil, nil + } + return r, err + } + + if o.Head { + r = ocimem.NewBytesReader(nil, desc) + } + + return r, err +} diff --git a/registry/rate-limits.go b/registry/rate-limits.go index eb9e331..e5c52db 100644 --- a/registry/rate-limits.go +++ b/registry/rate-limits.go @@ -4,7 +4,6 @@ import ( "net/http" "time" - "cuelabs.dev/go/oci/ociregistry/ociclient" "golang.org/x/time/rate" ) @@ -14,13 +13,13 @@ var ( } ) -// an implementation of [ociclient.HTTPDoer] that transparently adds a total requests rate limit and 429-retrying behavior -type rateLimitedRetryingDoer struct { - doer ociclient.HTTPDoer - limiter *rate.Limiter +// an implementation of [net/http.RoundTripper] that transparently adds a total requests rate limit and 429-retrying behavior +type rateLimitedRetryingRoundTripper struct { + roundTripper http.RoundTripper + limiter *rate.Limiter } -func (d *rateLimitedRetryingDoer) Do(req *http.Request) (*http.Response, error) { +func (d *rateLimitedRetryingRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { requestRetryLimiter := rate.NewLimiter(rate.Every(time.Second), 1) // cap request retries at once per second firstTry := true ctx := req.Context() @@ -32,23 +31,32 @@ func (d *rateLimitedRetryingDoer) Do(req *http.Request) (*http.Response, error) return nil, err } - if !firstTry && req.GetBody != nil { - var err error - req.Body, err = req.GetBody() - if err != nil { - return nil, err + if !firstTry { + // https://pkg.go.dev/net/http#RoundTripper + // "RoundTrip should not modify the request, except for consuming and closing the Request's Body." + if req.Body != nil { + req.Body.Close() + } + req = req.Clone(ctx) + if req.GetBody != nil { + var err error + req.Body, err = req.GetBody() + if err != nil { + return nil, err + } } } firstTry = false - res, err := d.doer.Do(req) + // in theory, this RoundTripper we're invoking should close req.Body (per the RoundTripper contract), so we shouldn't have to 🤞 + res, err := d.roundTripper.RoundTrip(req) if err != nil { return nil, err } // TODO 503 should probably result in at least one or two auto-retries (especially with the automatic retry delay this injects) if res.StatusCode == 429 { - // satisfy the big scary warning on https://pkg.go.dev/net/http#Client.Do about the downsides of failing to Close the response body + // satisfy the big scary warnings on https://pkg.go.dev/net/http#RoundTripper and https://pkg.go.dev/net/http#Client.Do about the downsides of failing to Close the response body if err := res.Body.Close(); err != nil { return nil, err } diff --git a/registry/read-helpers.go b/registry/read-helpers.go index 6fdac08..03d796f 100644 --- a/registry/read-helpers.go +++ b/registry/read-helpers.go @@ -20,6 +20,8 @@ func readJSONHelper(r ociregistry.BlobReader, v interface{}) error { return err } + // TODO if desc.Data != nil and len() == desc.Size, we should probably check/use that? 👀 + // make sure we can't possibly read (much) more than we're supposed to limited := &io.LimitedReader{ R: r, diff --git a/registry/synthesize-index.go b/registry/synthesize-index.go index af79a48..ee2c8d9 100644 --- a/registry/synthesize-index.go +++ b/registry/synthesize-index.go @@ -2,47 +2,29 @@ package registry import ( "context" - "errors" "fmt" - "strings" "github.com/docker-library/bashbrew/architecture" "cuelabs.dev/go/oci/ociregistry" + "cuelabs.dev/go/oci/ociregistry/ocimem" ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) // returns a synthesized [ocispec.Index] object for the given reference that includes automatically pulling up [ocispec.Platform] objects for entries missing them plus annotations for bashbrew architecture ([AnnotationBashbrewArch]) and where to find the "upstream" object if it needs to be copied/pulled ([ocispec.AnnotationRefName]) func SynthesizeIndex(ctx context.Context, ref Reference) (*ocispec.Index, error) { - // consider making this a full ociregistry.Interface object? GetManifest(digest) not returning an object with that digest would certainly be Weird though so maybe that's a misguided idea (with very minimal actual benefit, at least right now) - client, err := Client(ref.Host, nil) if err != nil { return nil, fmt.Errorf("%s: failed getting client: %w", ref, err) } - var r ociregistry.BlobReader = nil - if ref.Digest != "" { - r, err = client.GetManifest(ctx, ref.Repository, ref.Digest) - } else { - tag := ref.Tag - if tag == "" { - tag = "latest" - } - r, err = client.GetTag(ctx, ref.Repository, tag) - } + r, err := Lookup(ctx, ref, nil) if err != nil { - // https://github.com/cue-labs/oci/issues/26 - if errors.Is(err, ociregistry.ErrBlobUnknown) || - errors.Is(err, ociregistry.ErrManifestUnknown) || - errors.Is(err, ociregistry.ErrNameUnknown) || - strings.HasPrefix(err.Error(), "404 ") || - // 401 often means "repository not found" (due to the nature of public/private mixing on Hub and the fact that ociauth definitely handled any possible authentication for us, so if we're still getting 401 it's unavoidable and might as well be 404) - strings.HasPrefix(err.Error(), "401 ") { - return nil, nil - } return nil, fmt.Errorf("%s: failed GET: %w", ref, err) } + if r == nil { + return nil, nil + } defer r.Close() desc := r.Descriptor() @@ -127,6 +109,11 @@ func SynthesizeIndex(ctx context.Context, ref Reference) (*ocispec.Index, error) } } + // TODO if m.Size > 2048 { + // make sure we don't return any (big) data fields, now that we know we don't need them for sure (they might exist in the index we queried, but they're also used as an implementation detail in our registry cache code to store the original upstream data) + m.Data = nil + // } + index.Manifests[i] = m seen[string(m.Digest)] = &index.Manifests[i] i++ @@ -158,9 +145,13 @@ func normalizeManifestPlatform(ctx context.Context, m *ocispec.Descriptor, r oci case ocispec.MediaTypeImageManifest, mediaTypeDockerImageManifest: var err error if r == nil { - r, err = client.GetManifest(ctx, ref.Repository, m.Digest) - if err != nil { - return err + if m.Data != nil && int64(len(m.Data)) == m.Size { + r = ocimem.NewBytesReader(m.Data, *m) + } else { + r, err = client.GetManifest(ctx, ref.Repository, m.Digest) + if err != nil { + return err + } } defer r.Close() } @@ -172,9 +163,14 @@ func normalizeManifestPlatform(ctx context.Context, m *ocispec.Descriptor, r oci switch manifest.Config.MediaType { case ocispec.MediaTypeImageConfig, mediaTypeDockerImageConfig: - r, err := client.GetBlob(ctx, ref.Repository, manifest.Config.Digest) - if err != nil { - return err + var r ociregistry.BlobReader + if manifest.Config.Data != nil && int64(len(manifest.Config.Data)) == manifest.Config.Size { + r = ocimem.NewBytesReader(manifest.Config.Data, manifest.Config) + } else { + r, err = client.GetBlob(ctx, ref.Repository, manifest.Config.Digest) + if err != nil { + return err + } } defer r.Close()