diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..cb4ab61d2 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "go-ethereum"] + path = go-ethereum + url = https://github.com/ethereum/go-ethereum diff --git a/account_proof.go b/account_proof.go index e6a8493f9..54079c2e7 100644 --- a/account_proof.go +++ b/account_proof.go @@ -24,7 +24,7 @@ import ( "fmt" "strings" - "github.com/ethereum/go-ethereum/rlp" + "github.com/onflow/flow-go-sdk/encoding/rlp" ) // AccountProofNonceMinLenBytes is the minimum length of account proof nonces in bytes. diff --git a/encoding/.github/CODEOWNERS b/encoding/.github/CODEOWNERS new file mode 100644 index 000000000..89ddbc170 --- /dev/null +++ b/encoding/.github/CODEOWNERS @@ -0,0 +1,24 @@ +# Lines starting with '#' are comments. +# Each line is a file pattern followed by one or more owners. + +accounts/usbwallet @karalabe +accounts/scwallet @gballet +accounts/abi @gballet @MariusVanDerWijden +cmd/clef @holiman +cmd/puppeth @karalabe +consensus @karalabe +core/ @karalabe @holiman @rjl493456442 +eth/ @karalabe @holiman @rjl493456442 +eth/catalyst/ @gballet +eth/tracers/ @s1na +graphql/ @gballet @s1na +les/ @zsfelfoldi @rjl493456442 +light/ @zsfelfoldi @rjl493456442 +mobile/ @karalabe @ligi +node/ @fjl +p2p/ @fjl @zsfelfoldi +rpc/ @fjl @holiman +p2p/simulations @fjl +p2p/protocols @fjl +p2p/testing @fjl +signer/ @holiman diff --git a/encoding/.github/CONTRIBUTING.md b/encoding/.github/CONTRIBUTING.md new file mode 100644 index 000000000..a08542df2 --- /dev/null +++ b/encoding/.github/CONTRIBUTING.md @@ -0,0 +1,40 @@ +# Contributing + +Thank you for considering to help out with the source code! We welcome +contributions from anyone on the internet, and are grateful for even the +smallest of fixes! + +If you'd like to contribute to go-ethereum, please fork, fix, commit and send a +pull request for the maintainers to review and merge into the main code base. If +you wish to submit more complex changes though, please check up with the core +devs first on [our gitter channel](https://gitter.im/ethereum/go-ethereum) to +ensure those changes are in line with the general philosophy of the project +and/or get some early feedback which can make both your efforts much lighter as +well as our review and merge procedures quick and simple. + +## Coding guidelines + +Please make sure your contributions adhere to our coding guidelines: + + * Code must adhere to the official Go +[formatting](https://golang.org/doc/effective_go.html#formatting) guidelines +(i.e. uses [gofmt](https://golang.org/cmd/gofmt/)). + * Code must be documented adhering to the official Go +[commentary](https://golang.org/doc/effective_go.html#commentary) guidelines. + * Pull requests need to be based on and opened against the `master` branch. + * Commit messages should be prefixed with the package(s) they modify. + * E.g. "eth, rpc: make trace configs optional" + +## Can I have feature X + +Before you submit a feature request, please check and make sure that it isn't +possible through some other means. The JavaScript-enabled console is a powerful +feature in the right hands. Please check our +[Geth documentation page](https://geth.ethereum.org/docs/) for more info +and help. + +## Configuration, dependencies, and tests + +Please see the [Developers' Guide](https://geth.ethereum.org/docs/developers/devguide) +for more details on configuring your environment, managing project dependencies +and testing procedures. diff --git a/encoding/.github/ISSUE_TEMPLATE/bug.md b/encoding/.github/ISSUE_TEMPLATE/bug.md new file mode 100644 index 000000000..2aa2c48a6 --- /dev/null +++ b/encoding/.github/ISSUE_TEMPLATE/bug.md @@ -0,0 +1,30 @@ +--- +name: Report a bug +about: Something with go-ethereum is not working as expected +title: '' +labels: 'type:bug' +assignees: '' +--- + +#### System information + +Geth version: `geth version` +OS & Version: Windows/Linux/OSX +Commit hash : (if `develop`) + +#### Expected behaviour + + +#### Actual behaviour + + +#### Steps to reproduce the behaviour + + +#### Backtrace + +```` +[backtrace] +```` + +When submitting logs: please submit them as text and not screenshots. \ No newline at end of file diff --git a/encoding/.github/ISSUE_TEMPLATE/feature.md b/encoding/.github/ISSUE_TEMPLATE/feature.md new file mode 100644 index 000000000..aacd885f9 --- /dev/null +++ b/encoding/.github/ISSUE_TEMPLATE/feature.md @@ -0,0 +1,17 @@ +--- +name: Request a feature +about: Report a missing feature - e.g. as a step before submitting a PR +title: '' +labels: 'type:feature' +assignees: '' +--- + +# Rationale + +Why should this feature exist? +What are the use-cases? + +# Implementation + +Do you have ideas regarding the implementation of this feature? +Are you willing to implement this feature? \ No newline at end of file diff --git a/encoding/.github/ISSUE_TEMPLATE/question.md b/encoding/.github/ISSUE_TEMPLATE/question.md new file mode 100644 index 000000000..8f460ab55 --- /dev/null +++ b/encoding/.github/ISSUE_TEMPLATE/question.md @@ -0,0 +1,9 @@ +--- +name: Ask a question +about: Something is unclear +title: '' +labels: 'type:docs' +assignees: '' +--- + +This should only be used in very rare cases e.g. if you are not 100% sure if something is a bug or asking a question that leads to improving the documentation. For general questions please use [discord](https://discord.gg/nthXNEv) or the Ethereum stack exchange at https://ethereum.stackexchange.com. diff --git a/encoding/.github/no-response.yml b/encoding/.github/no-response.yml new file mode 100644 index 000000000..903d4ce85 --- /dev/null +++ b/encoding/.github/no-response.yml @@ -0,0 +1,11 @@ +# Number of days of inactivity before an Issue is closed for lack of response +daysUntilClose: 30 +# Label requiring a response +responseRequiredLabel: "need:more-information" +# Comment to post when closing an Issue for lack of response. Set to `false` to disable +closeComment: > + This issue has been automatically closed because there has been no response + to our request for more information from the original author. With only the + information that is currently in the issue, we don't have enough information + to take action. Please reach out if you have more relevant information or + answers to our questions so that we can investigate further. diff --git a/encoding/.github/stale.yml b/encoding/.github/stale.yml new file mode 100644 index 000000000..6d921cc79 --- /dev/null +++ b/encoding/.github/stale.yml @@ -0,0 +1,17 @@ +# Number of days of inactivity before an issue becomes stale +daysUntilStale: 366 +# Number of days of inactivity before a stale issue is closed +daysUntilClose: 42 +# Issues with these labels will never be considered stale +exemptLabels: + - pinned + - security +# Label to use when marking an issue as stale +staleLabel: "status:inactive" +# Comment to post when marking an issue as stale. Set to `false` to disable +markComment: > + This issue has been automatically marked as stale because it has not had + recent activity. It will be closed if no further activity occurs. Thank you + for your contributions. +# Comment to post when closing a stale issue. Set to `false` to disable +closeComment: false diff --git a/encoding/AUTHORS b/encoding/AUTHORS new file mode 100644 index 000000000..151c85016 --- /dev/null +++ b/encoding/AUTHORS @@ -0,0 +1,591 @@ +# This is the official list of go-ethereum authors for copyright purposes. + +6543 <6543@obermui.de> +a e r t h +Aaron Buchwald +Abel Nieto +Adam Babik +Adam Schmideg +Aditya +Aditya Arora +Adrià Cidre +Afanasii Kurakin +Afri Schoedon <5chdn@users.noreply.github.com> +Agustin Armellini Fischer +Ahyun +Airead +Alan Chen +Alejandro Isaza +Aleksey Smyrnov +Ales Katona +Alex Beregszaszi +Alex Leverington +Alex Mazalov +Alex Pozhilenkov +Alex Prut <1648497+alexprut@users.noreply.github.com> +Alex Wu +Alexander van der Meij +Alexander Yastrebov +Alexandre Van de Sande +Alexey Akhunov +Alexey Shekhirin +alexwang <39109351+dipingxian2@users.noreply.github.com> +Ali Atiia <42751398+aliatiia@users.noreply.github.com> +Ali Hajimirza +am2rican5 +AmitBRD <60668103+AmitBRD@users.noreply.github.com> +Anatole <62328077+a2br@users.noreply.github.com> +Andrea Franz +Andrei Maiboroda +Andrey Petrov +ANOTHEL +Antoine Rondelet +Antoine Toulme +Anton Evangelatov +Antonio Salazar Cardozo +Arba Sasmoyo +Armani Ferrante +Armin Braun +Aron Fischer +atsushi-ishibashi +Austin Roberts +ayeowch +b00ris +b1ackd0t +bailantaotao +baizhenxuan +Balaji Shetty Pachai <32358081+balajipachai@users.noreply.github.com> +Balint Gabor +baptiste-b-pegasys <85155432+baptiste-b-pegasys@users.noreply.github.com> +Bas van Kervel +Benjamin Brent +benma +Benoit Verkindt +Binacs +bloonfield +Bo +Bo Ye +Bob Glickstein +Boqin Qin +Brandon Harden +Brent +Brian Schroeder +Bruno Škvorc +C. Brown +Caesar Chad +Casey Detrio +CDsigma +Ceelog +Ceyhun Onur +chabashilah +changhong +Chase Wright +Chen Quan +Cheng Li +chenglin <910372762@qq.com> +chenyufeng +Chris Pacia +Chris Ziogas +Christian Muehlhaeuser +Christoph Jentzsch +chuwt +cong +Connor Stein +Corey Lin <514971757@qq.com> +courtier +cpusoft +Crispin Flowerday +croath +cui <523516579@qq.com> +Dan DeGreef +Dan Kinsley +Dan Sosedoff +Daniel A. Nagy +Daniel Perez +Daniel Sloof +Darioush Jalali +Darrel Herbst +Dave Appleton +Dave McGregor +David Cai +David Huie +Denver +Derek Chiang +Derek Gottfrid +Di Peng +Diederik Loerakker +Diego Siqueira +Diep Pham +dipingxian2 <39109351+dipingxian2@users.noreply.github.com> +divergencetech <94644849+divergencetech@users.noreply.github.com> +dm4 +Dmitrij Koniajev +Dmitry Shulyak +Dmitry Zenovich +Domino Valdano +Dragan Milic +dragonvslinux <35779158+dragononcrypto@users.noreply.github.com> +Edgar Aroutiounian +Eduard S +Egon Elbre +Elad +Eli +Elias Naur +Elliot Shepherd +Emil +emile +Emmanuel T Odeke +Eng Zer Jun +Enrique Fynn +Enrique Ortiz +EOS Classic +Erichin +Ernesto del Toro +Ethan Buchman +ethersphere +Eugene Lepeico +Eugene Valeyev +Evangelos Pappas +Everton Fraga +Evgeny +Evgeny Danilenko <6655321@bk.ru> +evgk +Evolution404 <35091674+Evolution404@users.noreply.github.com> +EXEC +Fabian Vogelsteller +Fabio Barone +Fabio Berger +FaceHo +Felipe Strozberg <48066928+FelStroz@users.noreply.github.com> +Felix Lange +Ferenc Szabo +ferhat elmas +Ferran Borreguero +Fiisio +Fire Man <55934298+basdevelop@users.noreply.github.com> +flowerofdream <775654398@qq.com> +fomotrader <82184770+fomotrader@users.noreply.github.com> +ForLina <471133417@qq.com> +Frank Szendzielarz <33515470+FrankSzendzielarz@users.noreply.github.com> +Frank Wang +Franklin +Furkan KAMACI +Fuyang Deng +GagziW +Gary Rong +Gautam Botrel +George Ornbo +Giuseppe Bertone +Greg Colvin +Gregg Dourgarian +Gregory Markou <16929357+GregTheGreek@users.noreply.github.com> +Guifel +Guilherme Salgado +Guillaume Ballet +Guillaume Nicolas +GuiltyMorishita +Guruprasad Kamath <48196632+gurukamath@users.noreply.github.com> +Gus +Gustav Simonsson +Gísli Kristjánsson +Ha ĐANG +HackyMiner +hadv +Hanjiang Yu +Hao Bryan Cheng +Hao Duan +HAOYUatHZ <37070449+HAOYUatHZ@users.noreply.github.com> +Harry Dutton +haryu703 <34744512+haryu703@users.noreply.github.com> +Hendrik Hofstadt +Henning Diedrich +henopied <13500516+henopied@users.noreply.github.com> +hero5512 +holisticode +Hongbin Mao +Hsien-Tang Kao +hsyodyssey <47173566+hsyodyssey@users.noreply.github.com> +Husam Ibrahim <39692071+HusamIbrahim@users.noreply.github.com> +Hwanjo Heo <34005989+hwanjo@users.noreply.github.com> +hydai +Hyung-Kyu Hqueue Choi +Håvard Anda Estensen +Ian Macalinao +Ian Norden +icodezjb +Ikko Ashimine +Ilan Gitter <8359193+gitteri@users.noreply.github.com> +ImanSharaf <78227895+ImanSharaf@users.noreply.github.com> +Isidoro Ghezzi +Iskander (Alex) Sharipov +Ivan Bogatyy +Ivan Daniluk +Ivo Georgiev +jacksoom +Jae Kwon +James Prestwich <10149425+prestwich@users.noreply.github.com> +Jamie Pitts +Janoš Guljaš +Jared Wasinger +Jason Carver +Javier Peletier +Javier Sagredo +Jay +Jay Guo +Jaynti Kanani +Jeff Prestes +Jeff R. Allen +Jeff Wentworth +Jeffery Robert Walsh +Jeffrey Wilcke +Jens Agerberg +Jeremy McNevin +Jeremy Schlatter +Jerzy Lasyk +Jesse Tane +Jia Chenhui +Jim McDonald +jk-jeongkyun <45347815+jeongkyun-oh@users.noreply.github.com> +jkcomment +JoeGruffins <34998433+JoeGruffins@users.noreply.github.com> +Joel Burget +John C. Vernaleo +John Difool +Johns Beharry +Jonas +Jonathan Brown +Jonathan Chappelow +Jonathan Gimeno +JoranHonig +Jordan Krage +Jorropo +Joseph Chow +Joshua Colvin +Joshua Gutow +jovijovi +jtakalai +JU HYEONG PARK +Julian Y +Justin Clark-Casey +Justin Drake +Justus +Kawashima <91420903+sscodereth@users.noreply.github.com> +ken10100147 +Kenji Siu +Kenso Trabing +Kevin +kevin.xu +KibGzr +kiel barry +kilic +kimmylin <30611210+kimmylin@users.noreply.github.com> +Kitten King <53072918+kittenking@users.noreply.github.com> +knarfeh +Kobi Gurkan +komika +Konrad Feldmeier +Kris Shinn +Kristofer Peterson +Kumar Anirudha +Kurkó Mihály +Kushagra Sharma +Kwuaint <34888408+kwuaint@users.noreply.github.com> +Kyuntae Ethan Kim +Lee Bousfield +Lefteris Karapetsas +Leif Jurvetson +Leo Shklovskii +LeoLiao +Lewis Marshall +lhendre +Li Dongwei +Liang Ma +Liang ZOU +libby kent +libotony +LieutenantRoger +ligi +Lio李欧 +lmittmann +Lorenzo Manacorda +Louis Holbrook +Luca Zeug +Lucas Hendren +lzhfromustc <43191155+lzhfromustc@users.noreply.github.com> +Magicking +manlio +Maran Hidskes +Marek Kotewicz +Mariano Cortesi +Marius van der Wijden +Mark +Mark Rushakoff +mark.lin +Martin Alex Philip Dawson +Martin Holst Swende +Martin Klepsch +Martin Lundfall +Martin Michlmayr +Martin Redmond <21436+reds@users.noreply.github.com> +Mason Fischer +Mateusz Morusiewicz <11313015+Ruteri@users.noreply.github.com> +Mats Julian Olsen +Matt Garnett <14004106+lightclient@users.noreply.github.com> +Matt K <1036969+mkrump@users.noreply.github.com> +Matthew Di Ferrante +Matthew Halpern +Matthew Wampler-Doty +Max Sistemich +Maxim Zhiburt +Maximilian Meister +me020523 +Melvin Junhee Woo +meowsbits +Micah Zoltu +Michael Forney +Michael Riabzev +Michael Ruminer +michael1011 +Miguel Mota +Mike Burr +Mikhail Mikheev +milesvant +Miro +Miya Chen +Mohanson +mr_franklin +Mudit Gupta +Mymskmkt <1847234666@qq.com> +Nalin Bhardwaj +Natsu Kagami +Nchinda Nchinda +nebojsa94 +necaremus +nedifi <103940716+nedifi@users.noreply.github.com> +needkane <604476380@qq.com> +Nguyen Kien Trung +Nguyen Sy Thanh Son +Nic Jansma +Nick Dodson +Nick Johnson +Nicolas Feignon +Nicolas Guillaume +Nikita Kozhemyakin +Nikola Madjarevic +Nilesh Trivedi +Nimrod Gutman +Nishant Das +njupt-moon <1015041018@njupt.edu.cn> +nkbai +noam-alchemy <76969113+noam-alchemy@users.noreply.github.com> +nobody +Noman +nujabes403 +Nye Liu +Oleg Kovalov +Oli Bye +Oliver Tale-Yazdi +Olivier Hervieu +Or Neeman +Osoro Bironga +Osuke +Pantelis Peslis +Pascal Dierich +Patrick O'Grady +Pau +Paul Berg +Paul Litvak +Paul-Armand Verhaegen +Paulo L F Casaretto +Paweł Bylica +Pedro Gomes +Pedro Pombeiro +Peter Broadhurst +peter cresswell +Peter Pratscher +Peter Simard +Petr Mikusek +Philip Schlump +Pierre Neter +Pierre R +piersy +PilkyuJung +Piotr Dyraga +ploui <64719999+ploui@users.noreply.github.com> +Preston Van Loon +Prince Sinha +Péter Szilágyi +qd-ethan <31876119+qdgogogo@users.noreply.github.com> +Qian Bin +Quest Henkart +Rachel Franks +Rafael Matias +Raghav Sood +Ralph Caraveo +Ramesh Nair +rangzen +reinerRubin +Rene Lubov <41963722+renaynay@users.noreply.github.com> +rhaps107 +Ricardo Catalinas Jiménez +Ricardo Domingos +Richard Hart +Rick +RJ Catalano +Rob +Rob Mulholand +Robert Zaremba +Roc Yu +Roman Mazalov <83914728+gopherxyz@users.noreply.github.com> +Ross <9055337+Chadsr@users.noreply.github.com> +Runchao Han +Russ Cox +Ryan Schneider +ryanc414 +Rémy Roy +S. Matthew English +salanfe +Sam <39165351+Xia-Sam@users.noreply.github.com> +Sammy Libre <7374093+sammy007@users.noreply.github.com> +Samuel Marks +sanskarkhare +Sarlor +Sasuke1964 +Satpal <28562234+SatpalSandhu61@users.noreply.github.com> +Saulius Grigaitis +Sean +Serhat Şevki Dinçer +Shane Bammel +shawn <36943337+lxex@users.noreply.github.com> +shigeyuki azuchi +Shihao Xia +Shiming +Shintaro Kaneko +shiqinfeng1 <150627601@qq.com> +Shuai Qi +Shude Li +Shunsuke Watanabe +silence +Simon Jentzsch +Sina Mahmoodi <1591639+s1na@users.noreply.github.com> +sixdays +SjonHortensius +Slava Karpenko +slumber1122 +Smilenator +soc1c +Sorin Neacsu +Sparty +Stein Dekker +Steve Gattuso +Steve Ruckdashel +Steve Waldman +Steven E. Harris +Steven Roose +stompesi +stormpang +sunxiaojun2014 +Suriyaa Sundararuban +Sylvain Laurent +Taeik Lim +tamirms +Tangui Clairet +Tatsuya Shimoda +Taylor Gerring +TColl <38299499+TColl@users.noreply.github.com> +terasum +tgyKomgo <52910426+tgyKomgo@users.noreply.github.com> +Thad Guidry +Thomas Bocek +thomasmodeneis +thumb8432 +Ti Zhou +tia-99 <67107070+tia-99@users.noreply.github.com> +Tim Cooijmans +Tobias Hildebrandt <79341166+tobias-hildebrandt@users.noreply.github.com> +Tosh Camille +tsarpaul +Tyler Chambers <2775339+tylerchambers@users.noreply.github.com> +tzapu +ucwong +uji <49834542+uji@users.noreply.github.com> +ult-bobonovski +Valentin Trinqué +Valentin Wüstholz +Vedhavyas Singareddi +Victor Farazdagi +Victor Tran +Vie +Viktor Trón +Ville Sundell +vim88 +Vincent G +Vincent Serpoul +Vinod Damle +Vitalik Buterin +Vitaly Bogdanov +Vitaly V +Vivek Anand +Vlad Bokov +Vlad Gluhovsky +Ward Bradt +Water <44689567+codeoneline@users.noreply.github.com> +wbt +weimumu <934657014@qq.com> +Wenbiao Zheng +Wenshao Zhong +Will Villanueva +William Morriss +William Setzer +williambannas +wuff1996 <33193253+wuff1996@users.noreply.github.com> +Wuxiang +Xiaobing Jiang +xiekeyang +xincaosu +xinluyin <31590468+xinluyin@users.noreply.github.com> +Xudong Liu <33193253+r1cs@users.noreply.github.com> +xwjack +yahtoo +Yang Hau +YaoZengzeng +YH-Zhou +Yihau Chen +Yohann Léon +Yoichi Hirai +Yole <007yuyue@gmail.com> +Yondon Fu +YOSHIDA Masanori +yoza +yumiel yoomee1313 +Yusup +yutianwu +ywzqwwt <39263032+ywzqwwt@users.noreply.github.com> +zaccoding +Zach +Zachinquarantine +zah +Zahoor Mohamed +Zak Cole +zcheng9 +zer0to0ne <36526113+zer0to0ne@users.noreply.github.com> +zgfzgf <48779939+zgfzgf@users.noreply.github.com> +Zhang Zhuo +zhangsoledad <787953403@qq.com> +zhaochonghe <41711151+zhaochonghe@users.noreply.github.com> +Zhenguo Niu +zhiqiangxu <652732310@qq.com> +Zhou Zhiyao +Ziyuan Zhong +Zoe Nolan +Zou Guangxian +Zsolt Felföldi +Łukasz Kurowski +Łukasz Zimnoch +ΞTHΞЯSPHΞЯΞ <{viktor.tron,nagydani,zsfelfoldi}@gmail.com> +Максим Чусовлянов +大彬 +沉风 +贺鹏飞 +陈佳 +유용환 <33824408+eric-yoo@users.noreply.github.com> diff --git a/encoding/COPYING b/encoding/COPYING new file mode 100644 index 000000000..f288702d2 --- /dev/null +++ b/encoding/COPYING @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/encoding/COPYING.LESSER b/encoding/COPYING.LESSER new file mode 100644 index 000000000..65c5ca88a --- /dev/null +++ b/encoding/COPYING.LESSER @@ -0,0 +1,165 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + + This version of the GNU Lesser General Public License incorporates +the terms and conditions of version 3 of the GNU General Public +License, supplemented by the additional permissions listed below. + + 0. Additional Definitions. + + As used herein, "this License" refers to version 3 of the GNU Lesser +General Public License, and the "GNU GPL" refers to version 3 of the GNU +General Public License. + + "The Library" refers to a covered work governed by this License, +other than an Application or a Combined Work as defined below. + + An "Application" is any work that makes use of an interface provided +by the Library, but which is not otherwise based on the Library. +Defining a subclass of a class defined by the Library is deemed a mode +of using an interface provided by the Library. + + A "Combined Work" is a work produced by combining or linking an +Application with the Library. The particular version of the Library +with which the Combined Work was made is also called the "Linked +Version". + + The "Minimal Corresponding Source" for a Combined Work means the +Corresponding Source for the Combined Work, excluding any source code +for portions of the Combined Work that, considered in isolation, are +based on the Application, and not on the Linked Version. + + The "Corresponding Application Code" for a Combined Work means the +object code and/or source code for the Application, including any data +and utility programs needed for reproducing the Combined Work from the +Application, but excluding the System Libraries of the Combined Work. + + 1. Exception to Section 3 of the GNU GPL. + + You may convey a covered work under sections 3 and 4 of this License +without being bound by section 3 of the GNU GPL. + + 2. Conveying Modified Versions. + + If you modify a copy of the Library, and, in your modifications, a +facility refers to a function or data to be supplied by an Application +that uses the facility (other than as an argument passed when the +facility is invoked), then you may convey a copy of the modified +version: + + a) under this License, provided that you make a good faith effort to + ensure that, in the event an Application does not supply the + function or data, the facility still operates, and performs + whatever part of its purpose remains meaningful, or + + b) under the GNU GPL, with none of the additional permissions of + this License applicable to that copy. + + 3. Object Code Incorporating Material from Library Header Files. + + The object code form of an Application may incorporate material from +a header file that is part of the Library. You may convey such object +code under terms of your choice, provided that, if the incorporated +material is not limited to numerical parameters, data structure +layouts and accessors, or small macros, inline functions and templates +(ten or fewer lines in length), you do both of the following: + + a) Give prominent notice with each copy of the object code that the + Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the object code with a copy of the GNU GPL and this license + document. + + 4. Combined Works. + + You may convey a Combined Work under terms of your choice that, +taken together, effectively do not restrict modification of the +portions of the Library contained in the Combined Work and reverse +engineering for debugging such modifications, if you also do each of +the following: + + a) Give prominent notice with each copy of the Combined Work that + the Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the Combined Work with a copy of the GNU GPL and this license + document. + + c) For a Combined Work that displays copyright notices during + execution, include the copyright notice for the Library among + these notices, as well as a reference directing the user to the + copies of the GNU GPL and this license document. + + d) Do one of the following: + + 0) Convey the Minimal Corresponding Source under the terms of this + License, and the Corresponding Application Code in a form + suitable for, and under terms that permit, the user to + recombine or relink the Application with a modified version of + the Linked Version to produce a modified Combined Work, in the + manner specified by section 6 of the GNU GPL for conveying + Corresponding Source. + + 1) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (a) uses at run time + a copy of the Library already present on the user's computer + system, and (b) will operate properly with a modified version + of the Library that is interface-compatible with the Linked + Version. + + e) Provide Installation Information, but only if you would otherwise + be required to provide such information under section 6 of the + GNU GPL, and only to the extent that such information is + necessary to install and execute a modified version of the + Combined Work produced by recombining or relinking the + Application with a modified version of the Linked Version. (If + you use option 4d0, the Installation Information must accompany + the Minimal Corresponding Source and Corresponding Application + Code. If you use option 4d1, you must provide the Installation + Information in the manner specified by section 6 of the GNU GPL + for conveying Corresponding Source.) + + 5. Combined Libraries. + + You may place library facilities that are a work based on the +Library side by side in a single library together with other library +facilities that are not Applications and are not covered by this +License, and convey such a combined library under terms of your +choice, if you do both of the following: + + a) Accompany the combined library with a copy of the same work based + on the Library, uncombined with any other library facilities, + conveyed under the terms of this License. + + b) Give prominent notice with the combined library that part of it + is a work based on the Library, and explaining where to find the + accompanying uncombined form of the same work. + + 6. Revised Versions of the GNU Lesser General Public License. + + The Free Software Foundation may publish revised and/or new versions +of the GNU Lesser General Public License from time to time. Such new +versions will be similar in spirit to the present version, but may +differ in detail to address new problems or concerns. + + Each version is given a distinguishing version number. If the +Library as you received it specifies that a certain numbered version +of the GNU Lesser General Public License "or any later version" +applies to it, you have the option of following the terms and +conditions either of that published version or of any later version +published by the Free Software Foundation. If the Library as you +received it does not specify a version number of the GNU Lesser +General Public License, you may choose any version of the GNU Lesser +General Public License ever published by the Free Software Foundation. + + If the Library as you received it specifies that a proxy can decide +whether future versions of the GNU Lesser General Public License shall +apply, that proxy's public statement of acceptance of any version is +permanent authorization for you to choose that version for the +Library. diff --git a/encoding/README.md b/encoding/README.md new file mode 100644 index 000000000..41a6757e4 --- /dev/null +++ b/encoding/README.md @@ -0,0 +1,30 @@ +## RLP Encoding + +This is a git subtree to the ethereum module found here: https://github.com/ethereum/go-ethereum/ + +Contents have been modified to only include the needed RLP encoding +functionality according to the licence found here: https://github.com/ethereum/go-ethereum/blob/master/COPYING.LESSER + +Copyright 2014 The go-ethereum Authors +This file is part of the go-ethereum library. + +The go-ethereum library is free software: you can redistribute it and/or modify +it under the terms of the GNU Lesser General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +The go-ethereum library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public License +along with the go-ethereum library. If not, see . + +## Updating + +You can use git subtree command to update from the upstream branch. + +``` +git subtree pull --prefix rlp https://github.com/ethereum/go-ethereum main --squash +``` diff --git a/encoding/common/big.go b/encoding/common/big.go new file mode 100644 index 000000000..65d4377bf --- /dev/null +++ b/encoding/common/big.go @@ -0,0 +1,30 @@ +// Copyright 2014 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package common + +import "math/big" + +// Common big integers often used +var ( + Big1 = big.NewInt(1) + Big2 = big.NewInt(2) + Big3 = big.NewInt(3) + Big0 = big.NewInt(0) + Big32 = big.NewInt(32) + Big256 = big.NewInt(256) + Big257 = big.NewInt(257) +) diff --git a/encoding/common/bitutil/bitutil.go b/encoding/common/bitutil/bitutil.go new file mode 100644 index 000000000..cd3e72169 --- /dev/null +++ b/encoding/common/bitutil/bitutil.go @@ -0,0 +1,188 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Adapted from: https://golang.org/src/crypto/cipher/xor.go + +// Package bitutil implements fast bitwise operations. +package bitutil + +import ( + "runtime" + "unsafe" +) + +const wordSize = int(unsafe.Sizeof(uintptr(0))) +const supportsUnaligned = runtime.GOARCH == "386" || runtime.GOARCH == "amd64" || runtime.GOARCH == "ppc64" || runtime.GOARCH == "ppc64le" || runtime.GOARCH == "s390x" + +// XORBytes xors the bytes in a and b. The destination is assumed to have enough +// space. Returns the number of bytes xor'd. +func XORBytes(dst, a, b []byte) int { + if supportsUnaligned { + return fastXORBytes(dst, a, b) + } + return safeXORBytes(dst, a, b) +} + +// fastXORBytes xors in bulk. It only works on architectures that support +// unaligned read/writes. +func fastXORBytes(dst, a, b []byte) int { + n := len(a) + if len(b) < n { + n = len(b) + } + w := n / wordSize + if w > 0 { + dw := *(*[]uintptr)(unsafe.Pointer(&dst)) + aw := *(*[]uintptr)(unsafe.Pointer(&a)) + bw := *(*[]uintptr)(unsafe.Pointer(&b)) + for i := 0; i < w; i++ { + dw[i] = aw[i] ^ bw[i] + } + } + for i := n - n%wordSize; i < n; i++ { + dst[i] = a[i] ^ b[i] + } + return n +} + +// safeXORBytes xors one by one. It works on all architectures, independent if +// it supports unaligned read/writes or not. +func safeXORBytes(dst, a, b []byte) int { + n := len(a) + if len(b) < n { + n = len(b) + } + for i := 0; i < n; i++ { + dst[i] = a[i] ^ b[i] + } + return n +} + +// ANDBytes ands the bytes in a and b. The destination is assumed to have enough +// space. Returns the number of bytes and'd. +func ANDBytes(dst, a, b []byte) int { + if supportsUnaligned { + return fastANDBytes(dst, a, b) + } + return safeANDBytes(dst, a, b) +} + +// fastANDBytes ands in bulk. It only works on architectures that support +// unaligned read/writes. +func fastANDBytes(dst, a, b []byte) int { + n := len(a) + if len(b) < n { + n = len(b) + } + w := n / wordSize + if w > 0 { + dw := *(*[]uintptr)(unsafe.Pointer(&dst)) + aw := *(*[]uintptr)(unsafe.Pointer(&a)) + bw := *(*[]uintptr)(unsafe.Pointer(&b)) + for i := 0; i < w; i++ { + dw[i] = aw[i] & bw[i] + } + } + for i := n - n%wordSize; i < n; i++ { + dst[i] = a[i] & b[i] + } + return n +} + +// safeANDBytes ands one by one. It works on all architectures, independent if +// it supports unaligned read/writes or not. +func safeANDBytes(dst, a, b []byte) int { + n := len(a) + if len(b) < n { + n = len(b) + } + for i := 0; i < n; i++ { + dst[i] = a[i] & b[i] + } + return n +} + +// ORBytes ors the bytes in a and b. The destination is assumed to have enough +// space. Returns the number of bytes or'd. +func ORBytes(dst, a, b []byte) int { + if supportsUnaligned { + return fastORBytes(dst, a, b) + } + return safeORBytes(dst, a, b) +} + +// fastORBytes ors in bulk. It only works on architectures that support +// unaligned read/writes. +func fastORBytes(dst, a, b []byte) int { + n := len(a) + if len(b) < n { + n = len(b) + } + w := n / wordSize + if w > 0 { + dw := *(*[]uintptr)(unsafe.Pointer(&dst)) + aw := *(*[]uintptr)(unsafe.Pointer(&a)) + bw := *(*[]uintptr)(unsafe.Pointer(&b)) + for i := 0; i < w; i++ { + dw[i] = aw[i] | bw[i] + } + } + for i := n - n%wordSize; i < n; i++ { + dst[i] = a[i] | b[i] + } + return n +} + +// safeORBytes ors one by one. It works on all architectures, independent if +// it supports unaligned read/writes or not. +func safeORBytes(dst, a, b []byte) int { + n := len(a) + if len(b) < n { + n = len(b) + } + for i := 0; i < n; i++ { + dst[i] = a[i] | b[i] + } + return n +} + +// TestBytes tests whether any bit is set in the input byte slice. +func TestBytes(p []byte) bool { + if supportsUnaligned { + return fastTestBytes(p) + } + return safeTestBytes(p) +} + +// fastTestBytes tests for set bits in bulk. It only works on architectures that +// support unaligned read/writes. +func fastTestBytes(p []byte) bool { + n := len(p) + w := n / wordSize + if w > 0 { + pw := *(*[]uintptr)(unsafe.Pointer(&p)) + for i := 0; i < w; i++ { + if pw[i] != 0 { + return true + } + } + } + for i := n - n%wordSize; i < n; i++ { + if p[i] != 0 { + return true + } + } + return false +} + +// safeTestBytes tests for set bits one byte at a time. It works on all +// architectures, independent if it supports unaligned read/writes or not. +func safeTestBytes(p []byte) bool { + for i := 0; i < len(p); i++ { + if p[i] != 0 { + return true + } + } + return false +} diff --git a/encoding/common/bitutil/bitutil_test.go b/encoding/common/bitutil/bitutil_test.go new file mode 100644 index 000000000..307bf731f --- /dev/null +++ b/encoding/common/bitutil/bitutil_test.go @@ -0,0 +1,221 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Adapted from: https://golang.org/src/crypto/cipher/xor_test.go + +package bitutil + +import ( + "bytes" + "testing" +) + +// Tests that bitwise XOR works for various alignments. +func TestXOR(t *testing.T) { + for alignP := 0; alignP < 2; alignP++ { + for alignQ := 0; alignQ < 2; alignQ++ { + for alignD := 0; alignD < 2; alignD++ { + p := make([]byte, 1023)[alignP:] + q := make([]byte, 1023)[alignQ:] + + for i := 0; i < len(p); i++ { + p[i] = byte(i) + } + for i := 0; i < len(q); i++ { + q[i] = byte(len(q) - i) + } + d1 := make([]byte, 1023+alignD)[alignD:] + d2 := make([]byte, 1023+alignD)[alignD:] + + XORBytes(d1, p, q) + safeXORBytes(d2, p, q) + if !bytes.Equal(d1, d2) { + t.Error("not equal", d1, d2) + } + } + } + } +} + +// Tests that bitwise AND works for various alignments. +func TestAND(t *testing.T) { + for alignP := 0; alignP < 2; alignP++ { + for alignQ := 0; alignQ < 2; alignQ++ { + for alignD := 0; alignD < 2; alignD++ { + p := make([]byte, 1023)[alignP:] + q := make([]byte, 1023)[alignQ:] + + for i := 0; i < len(p); i++ { + p[i] = byte(i) + } + for i := 0; i < len(q); i++ { + q[i] = byte(len(q) - i) + } + d1 := make([]byte, 1023+alignD)[alignD:] + d2 := make([]byte, 1023+alignD)[alignD:] + + ANDBytes(d1, p, q) + safeANDBytes(d2, p, q) + if !bytes.Equal(d1, d2) { + t.Error("not equal") + } + } + } + } +} + +// Tests that bitwise OR works for various alignments. +func TestOR(t *testing.T) { + for alignP := 0; alignP < 2; alignP++ { + for alignQ := 0; alignQ < 2; alignQ++ { + for alignD := 0; alignD < 2; alignD++ { + p := make([]byte, 1023)[alignP:] + q := make([]byte, 1023)[alignQ:] + + for i := 0; i < len(p); i++ { + p[i] = byte(i) + } + for i := 0; i < len(q); i++ { + q[i] = byte(len(q) - i) + } + d1 := make([]byte, 1023+alignD)[alignD:] + d2 := make([]byte, 1023+alignD)[alignD:] + + ORBytes(d1, p, q) + safeORBytes(d2, p, q) + if !bytes.Equal(d1, d2) { + t.Error("not equal") + } + } + } + } +} + +// Tests that bit testing works for various alignments. +func TestTest(t *testing.T) { + for align := 0; align < 2; align++ { + // Test for bits set in the bulk part + p := make([]byte, 1023)[align:] + p[100] = 1 + + if TestBytes(p) != safeTestBytes(p) { + t.Error("not equal") + } + // Test for bits set in the tail part + q := make([]byte, 1023)[align:] + q[len(q)-1] = 1 + + if TestBytes(q) != safeTestBytes(q) { + t.Error("not equal") + } + } +} + +// Benchmarks the potentially optimized XOR performance. +func BenchmarkFastXOR1KB(b *testing.B) { benchmarkFastXOR(b, 1024) } +func BenchmarkFastXOR2KB(b *testing.B) { benchmarkFastXOR(b, 2048) } +func BenchmarkFastXOR4KB(b *testing.B) { benchmarkFastXOR(b, 4096) } + +func benchmarkFastXOR(b *testing.B, size int) { + p, q := make([]byte, size), make([]byte, size) + + for i := 0; i < b.N; i++ { + XORBytes(p, p, q) + } +} + +// Benchmarks the baseline XOR performance. +func BenchmarkBaseXOR1KB(b *testing.B) { benchmarkBaseXOR(b, 1024) } +func BenchmarkBaseXOR2KB(b *testing.B) { benchmarkBaseXOR(b, 2048) } +func BenchmarkBaseXOR4KB(b *testing.B) { benchmarkBaseXOR(b, 4096) } + +func benchmarkBaseXOR(b *testing.B, size int) { + p, q := make([]byte, size), make([]byte, size) + + for i := 0; i < b.N; i++ { + safeXORBytes(p, p, q) + } +} + +// Benchmarks the potentially optimized AND performance. +func BenchmarkFastAND1KB(b *testing.B) { benchmarkFastAND(b, 1024) } +func BenchmarkFastAND2KB(b *testing.B) { benchmarkFastAND(b, 2048) } +func BenchmarkFastAND4KB(b *testing.B) { benchmarkFastAND(b, 4096) } + +func benchmarkFastAND(b *testing.B, size int) { + p, q := make([]byte, size), make([]byte, size) + + for i := 0; i < b.N; i++ { + ANDBytes(p, p, q) + } +} + +// Benchmarks the baseline AND performance. +func BenchmarkBaseAND1KB(b *testing.B) { benchmarkBaseAND(b, 1024) } +func BenchmarkBaseAND2KB(b *testing.B) { benchmarkBaseAND(b, 2048) } +func BenchmarkBaseAND4KB(b *testing.B) { benchmarkBaseAND(b, 4096) } + +func benchmarkBaseAND(b *testing.B, size int) { + p, q := make([]byte, size), make([]byte, size) + + for i := 0; i < b.N; i++ { + safeANDBytes(p, p, q) + } +} + +// Benchmarks the potentially optimized OR performance. +func BenchmarkFastOR1KB(b *testing.B) { benchmarkFastOR(b, 1024) } +func BenchmarkFastOR2KB(b *testing.B) { benchmarkFastOR(b, 2048) } +func BenchmarkFastOR4KB(b *testing.B) { benchmarkFastOR(b, 4096) } + +func benchmarkFastOR(b *testing.B, size int) { + p, q := make([]byte, size), make([]byte, size) + + for i := 0; i < b.N; i++ { + ORBytes(p, p, q) + } +} + +// Benchmarks the baseline OR performance. +func BenchmarkBaseOR1KB(b *testing.B) { benchmarkBaseOR(b, 1024) } +func BenchmarkBaseOR2KB(b *testing.B) { benchmarkBaseOR(b, 2048) } +func BenchmarkBaseOR4KB(b *testing.B) { benchmarkBaseOR(b, 4096) } + +func benchmarkBaseOR(b *testing.B, size int) { + p, q := make([]byte, size), make([]byte, size) + + for i := 0; i < b.N; i++ { + safeORBytes(p, p, q) + } +} + +var GloBool bool // Exported global will not be dead-code eliminated, at least not yet. + +// Benchmarks the potentially optimized bit testing performance. +func BenchmarkFastTest1KB(b *testing.B) { benchmarkFastTest(b, 1024) } +func BenchmarkFastTest2KB(b *testing.B) { benchmarkFastTest(b, 2048) } +func BenchmarkFastTest4KB(b *testing.B) { benchmarkFastTest(b, 4096) } + +func benchmarkFastTest(b *testing.B, size int) { + p := make([]byte, size) + a := false + for i := 0; i < b.N; i++ { + a = a != TestBytes(p) + } + GloBool = a // Use of benchmark "result" to prevent total dead code elimination. +} + +// Benchmarks the baseline bit testing performance. +func BenchmarkBaseTest1KB(b *testing.B) { benchmarkBaseTest(b, 1024) } +func BenchmarkBaseTest2KB(b *testing.B) { benchmarkBaseTest(b, 2048) } +func BenchmarkBaseTest4KB(b *testing.B) { benchmarkBaseTest(b, 4096) } + +func benchmarkBaseTest(b *testing.B, size int) { + p := make([]byte, size) + a := false + for i := 0; i < b.N; i++ { + a = a != safeTestBytes(p) + } + GloBool = a // Use of benchmark "result" to prevent total dead code elimination. +} diff --git a/encoding/common/bitutil/compress.go b/encoding/common/bitutil/compress.go new file mode 100644 index 000000000..c057cee4a --- /dev/null +++ b/encoding/common/bitutil/compress.go @@ -0,0 +1,170 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package bitutil + +import "errors" + +var ( + // errMissingData is returned from decompression if the byte referenced by + // the bitset header overflows the input data. + errMissingData = errors.New("missing bytes on input") + + // errUnreferencedData is returned from decompression if not all bytes were used + // up from the input data after decompressing it. + errUnreferencedData = errors.New("extra bytes on input") + + // errExceededTarget is returned from decompression if the bitset header has + // more bits defined than the number of target buffer space available. + errExceededTarget = errors.New("target data size exceeded") + + // errZeroContent is returned from decompression if a data byte referenced in + // the bitset header is actually a zero byte. + errZeroContent = errors.New("zero byte in input content") +) + +// The compression algorithm implemented by CompressBytes and DecompressBytes is +// optimized for sparse input data which contains a lot of zero bytes. Decompression +// requires knowledge of the decompressed data length. +// +// Compression works as follows: +// +// if data only contains zeroes, +// CompressBytes(data) == nil +// otherwise if len(data) <= 1, +// CompressBytes(data) == data +// otherwise: +// CompressBytes(data) == append(CompressBytes(nonZeroBitset(data)), nonZeroBytes(data)...) +// where +// nonZeroBitset(data) is a bit vector with len(data) bits (MSB first): +// nonZeroBitset(data)[i/8] && (1 << (7-i%8)) != 0 if data[i] != 0 +// len(nonZeroBitset(data)) == (len(data)+7)/8 +// nonZeroBytes(data) contains the non-zero bytes of data in the same order + +// CompressBytes compresses the input byte slice according to the sparse bitset +// representation algorithm. If the result is bigger than the original input, no +// compression is done. +func CompressBytes(data []byte) []byte { + if out := bitsetEncodeBytes(data); len(out) < len(data) { + return out + } + cpy := make([]byte, len(data)) + copy(cpy, data) + return cpy +} + +// bitsetEncodeBytes compresses the input byte slice according to the sparse +// bitset representation algorithm. +func bitsetEncodeBytes(data []byte) []byte { + // Empty slices get compressed to nil + if len(data) == 0 { + return nil + } + // One byte slices compress to nil or retain the single byte + if len(data) == 1 { + if data[0] == 0 { + return nil + } + return data + } + // Calculate the bitset of set bytes, and gather the non-zero bytes + nonZeroBitset := make([]byte, (len(data)+7)/8) + nonZeroBytes := make([]byte, 0, len(data)) + + for i, b := range data { + if b != 0 { + nonZeroBytes = append(nonZeroBytes, b) + nonZeroBitset[i/8] |= 1 << byte(7-i%8) + } + } + if len(nonZeroBytes) == 0 { + return nil + } + return append(bitsetEncodeBytes(nonZeroBitset), nonZeroBytes...) +} + +// DecompressBytes decompresses data with a known target size. If the input data +// matches the size of the target, it means no compression was done in the first +// place. +func DecompressBytes(data []byte, target int) ([]byte, error) { + if len(data) > target { + return nil, errExceededTarget + } + if len(data) == target { + cpy := make([]byte, len(data)) + copy(cpy, data) + return cpy, nil + } + return bitsetDecodeBytes(data, target) +} + +// bitsetDecodeBytes decompresses data with a known target size. +func bitsetDecodeBytes(data []byte, target int) ([]byte, error) { + out, size, err := bitsetDecodePartialBytes(data, target) + if err != nil { + return nil, err + } + if size != len(data) { + return nil, errUnreferencedData + } + return out, nil +} + +// bitsetDecodePartialBytes decompresses data with a known target size, but does +// not enforce consuming all the input bytes. In addition to the decompressed +// output, the function returns the length of compressed input data corresponding +// to the output as the input slice may be longer. +func bitsetDecodePartialBytes(data []byte, target int) ([]byte, int, error) { + // Sanity check 0 targets to avoid infinite recursion + if target == 0 { + return nil, 0, nil + } + // Handle the zero and single byte corner cases + decomp := make([]byte, target) + if len(data) == 0 { + return decomp, 0, nil + } + if target == 1 { + decomp[0] = data[0] // copy to avoid referencing the input slice + if data[0] != 0 { + return decomp, 1, nil + } + return decomp, 0, nil + } + // Decompress the bitset of set bytes and distribute the non zero bytes + nonZeroBitset, ptr, err := bitsetDecodePartialBytes(data, (target+7)/8) + if err != nil { + return nil, ptr, err + } + for i := 0; i < 8*len(nonZeroBitset); i++ { + if nonZeroBitset[i/8]&(1<= len(data) { + return nil, 0, errMissingData + } + if i >= len(decomp) { + return nil, 0, errExceededTarget + } + // Make sure the data is valid and push into the slot + if data[ptr] == 0 { + return nil, 0, errZeroContent + } + decomp[i] = data[ptr] + ptr++ + } + } + return decomp, ptr, nil +} diff --git a/encoding/common/bitutil/compress_test.go b/encoding/common/bitutil/compress_test.go new file mode 100644 index 000000000..be1e485b7 --- /dev/null +++ b/encoding/common/bitutil/compress_test.go @@ -0,0 +1,181 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package bitutil + +import ( + "bytes" + "math/rand" + "testing" + + "github.com/onflow/flow-go-sdk/encoding/common/hexutil" +) + +// Tests that data bitset encoding and decoding works and is bijective. +func TestEncodingCycle(t *testing.T) { + tests := []string{ + // Tests generated by go-fuzz to maximize code coverage + "0x000000000000000000", + "0xef0400", + "0xdf7070533534333636313639343638373532313536346c1bc33339343837313070706336343035336336346c65fefb3930393233383838ac2f65fefb", + "0x7b64000000", + "0x000034000000000000", + "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f0000000000000000000", + "0x4912385c0e7b64000000", + "0x000034000000000000000000000000000000", + "0x00", + "0x000003e834ff7f0000", + "0x0000", + "0x0000000000000000000000000000000000000000000000000000000000ff00", + "0x895f0c6a020f850c6a020f85f88df88d", + "0xdf7070533534333636313639343638373432313536346c1bc3315aac2f65fefb", + "0x0000000000", + "0xdf70706336346c65fefb", + "0x00006d643634000000", + "0xdf7070533534333636313639343638373532313536346c1bc333393438373130707063363430353639343638373532313536346c1bc333393438336336346c65fe", + } + for i, tt := range tests { + data := hexutil.MustDecode(tt) + + proc, err := bitsetDecodeBytes(bitsetEncodeBytes(data), len(data)) + if err != nil { + t.Errorf("test %d: failed to decompress compressed data: %v", i, err) + continue + } + if !bytes.Equal(data, proc) { + t.Errorf("test %d: compress/decompress mismatch: have %x, want %x", i, proc, data) + } + } +} + +// Tests that data bitset decoding and rencoding works and is bijective. +func TestDecodingCycle(t *testing.T) { + tests := []struct { + size int + input string + fail error + }{ + {size: 0, input: "0x"}, + + // Crashers generated by go-fuzz + {size: 0, input: "0x0020", fail: errUnreferencedData}, + {size: 0, input: "0x30", fail: errUnreferencedData}, + {size: 1, input: "0x00", fail: errUnreferencedData}, + {size: 2, input: "0x07", fail: errMissingData}, + {size: 1024, input: "0x8000", fail: errZeroContent}, + + // Tests generated by go-fuzz to maximize code coverage + {size: 29490, input: "0x343137343733323134333839373334323073333930783e3078333930783e70706336346c65303e", fail: errMissingData}, + {size: 59395, input: "0x00", fail: errUnreferencedData}, + {size: 52574, input: "0x70706336346c65c0de", fail: errExceededTarget}, + {size: 42264, input: "0x07", fail: errMissingData}, + {size: 52, input: "0xa5045bad48f4", fail: errExceededTarget}, + {size: 52574, input: "0xc0de", fail: errMissingData}, + {size: 52574, input: "0x"}, + {size: 29490, input: "0x34313734373332313433383937333432307333393078073034333839373334323073333930783e3078333937333432307333393078073061333930783e70706336346c65303e", fail: errMissingData}, + {size: 29491, input: "0x3973333930783e30783e", fail: errMissingData}, + + {size: 1024, input: "0x808080608080"}, + {size: 1024, input: "0x808470705e3632383337363033313434303137393130306c6580ef46806380635a80"}, + {size: 1024, input: "0x8080808070"}, + {size: 1024, input: "0x808070705e36346c6580ef46806380635a80"}, + {size: 1024, input: "0x80808046802680"}, + {size: 1024, input: "0x4040404035"}, + {size: 1024, input: "0x4040bf3ba2b3f684402d353234373438373934409fe5b1e7ada94ebfd7d0505e27be4035"}, + {size: 1024, input: "0x404040bf3ba2b3f6844035"}, + {size: 1024, input: "0x40402d35323437343837393440bfd7d0505e27be4035"}, + } + for i, tt := range tests { + data := hexutil.MustDecode(tt.input) + + orig, err := bitsetDecodeBytes(data, tt.size) + if err != tt.fail { + t.Errorf("test %d: failure mismatch: have %v, want %v", i, err, tt.fail) + } + if err != nil { + continue + } + if comp := bitsetEncodeBytes(orig); !bytes.Equal(comp, data) { + t.Errorf("test %d: decompress/compress mismatch: have %x, want %x", i, comp, data) + } + } +} + +// TestCompression tests that compression works by returning either the bitset +// encoded input, or the actual input if the bitset version is longer. +func TestCompression(t *testing.T) { + // Check the compression returns the bitset encoding is shorter + in := hexutil.MustDecode("0x4912385c0e7b64000000") + out := hexutil.MustDecode("0x80fe4912385c0e7b64") + + if data := CompressBytes(in); !bytes.Equal(data, out) { + t.Errorf("encoding mismatch for sparse data: have %x, want %x", data, out) + } + if data, err := DecompressBytes(out, len(in)); err != nil || !bytes.Equal(data, in) { + t.Errorf("decoding mismatch for sparse data: have %x, want %x, error %v", data, in, err) + } + // Check the compression returns the input if the bitset encoding is longer + in = hexutil.MustDecode("0xdf7070533534333636313639343638373532313536346c1bc33339343837313070706336343035336336346c65fefb3930393233383838ac2f65fefb") + out = hexutil.MustDecode("0xdf7070533534333636313639343638373532313536346c1bc33339343837313070706336343035336336346c65fefb3930393233383838ac2f65fefb") + + if data := CompressBytes(in); !bytes.Equal(data, out) { + t.Errorf("encoding mismatch for dense data: have %x, want %x", data, out) + } + if data, err := DecompressBytes(out, len(in)); err != nil || !bytes.Equal(data, in) { + t.Errorf("decoding mismatch for dense data: have %x, want %x, error %v", data, in, err) + } + // Check that decompressing a longer input than the target fails + if _, err := DecompressBytes([]byte{0xc0, 0x01, 0x01}, 2); err != errExceededTarget { + t.Errorf("decoding error mismatch for long data: have %v, want %v", err, errExceededTarget) + } +} + +// Crude benchmark for compressing random slices of bytes. +func BenchmarkEncoding1KBVerySparse(b *testing.B) { benchmarkEncoding(b, 1024, 0.0001) } +func BenchmarkEncoding2KBVerySparse(b *testing.B) { benchmarkEncoding(b, 2048, 0.0001) } +func BenchmarkEncoding4KBVerySparse(b *testing.B) { benchmarkEncoding(b, 4096, 0.0001) } + +func BenchmarkEncoding1KBSparse(b *testing.B) { benchmarkEncoding(b, 1024, 0.001) } +func BenchmarkEncoding2KBSparse(b *testing.B) { benchmarkEncoding(b, 2048, 0.001) } +func BenchmarkEncoding4KBSparse(b *testing.B) { benchmarkEncoding(b, 4096, 0.001) } + +func BenchmarkEncoding1KBDense(b *testing.B) { benchmarkEncoding(b, 1024, 0.1) } +func BenchmarkEncoding2KBDense(b *testing.B) { benchmarkEncoding(b, 2048, 0.1) } +func BenchmarkEncoding4KBDense(b *testing.B) { benchmarkEncoding(b, 4096, 0.1) } + +func BenchmarkEncoding1KBSaturated(b *testing.B) { benchmarkEncoding(b, 1024, 0.5) } +func BenchmarkEncoding2KBSaturated(b *testing.B) { benchmarkEncoding(b, 2048, 0.5) } +func BenchmarkEncoding4KBSaturated(b *testing.B) { benchmarkEncoding(b, 4096, 0.5) } + +func benchmarkEncoding(b *testing.B, bytes int, fill float64) { + // Generate a random slice of bytes to compress + random := rand.NewSource(0) // reproducible and comparable + + data := make([]byte, bytes) + bits := int(float64(bytes) * 8 * fill) + + for i := 0; i < bits; i++ { + idx := random.Int63() % int64(len(data)) + bit := uint(random.Int63() % 8) + data[idx] |= 1 << bit + } + // Reset the benchmark and measure encoding/decoding + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + bitsetDecodeBytes(bitsetEncodeBytes(data), len(data)) + } +} diff --git a/encoding/common/bytes.go b/encoding/common/bytes.go new file mode 100644 index 000000000..3ffb72e9d --- /dev/null +++ b/encoding/common/bytes.go @@ -0,0 +1,151 @@ +// Copyright 2014 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Package common contains various helper functions. +package common + +import ( + "encoding/hex" + "errors" + + "github.com/onflow/flow-go-sdk/encoding/common/hexutil" +) + +// FromHex returns the bytes represented by the hexadecimal string s. +// s may be prefixed with "0x". +func FromHex(s string) []byte { + if has0xPrefix(s) { + s = s[2:] + } + if len(s)%2 == 1 { + s = "0" + s + } + return Hex2Bytes(s) +} + +// CopyBytes returns an exact copy of the provided bytes. +func CopyBytes(b []byte) (copiedBytes []byte) { + if b == nil { + return nil + } + copiedBytes = make([]byte, len(b)) + copy(copiedBytes, b) + + return +} + +// has0xPrefix validates str begins with '0x' or '0X'. +func has0xPrefix(str string) bool { + return len(str) >= 2 && str[0] == '0' && (str[1] == 'x' || str[1] == 'X') +} + +// isHexCharacter returns bool of c being a valid hexadecimal. +func isHexCharacter(c byte) bool { + return ('0' <= c && c <= '9') || ('a' <= c && c <= 'f') || ('A' <= c && c <= 'F') +} + +// isHex validates whether each byte is valid hexadecimal string. +func isHex(str string) bool { + if len(str)%2 != 0 { + return false + } + for _, c := range []byte(str) { + if !isHexCharacter(c) { + return false + } + } + return true +} + +// Bytes2Hex returns the hexadecimal encoding of d. +func Bytes2Hex(d []byte) string { + return hex.EncodeToString(d) +} + +// Hex2Bytes returns the bytes represented by the hexadecimal string str. +func Hex2Bytes(str string) []byte { + h, _ := hex.DecodeString(str) + return h +} + +// Hex2BytesFixed returns bytes of a specified fixed length flen. +func Hex2BytesFixed(str string, flen int) []byte { + h, _ := hex.DecodeString(str) + if len(h) == flen { + return h + } + if len(h) > flen { + return h[len(h)-flen:] + } + hh := make([]byte, flen) + copy(hh[flen-len(h):flen], h) + return hh +} + +// ParseHexOrString tries to hexdecode b, but if the prefix is missing, it instead just returns the raw bytes +func ParseHexOrString(str string) ([]byte, error) { + b, err := hexutil.Decode(str) + if errors.Is(err, hexutil.ErrMissingPrefix) { + return []byte(str), nil + } + return b, err +} + +// RightPadBytes zero-pads slice to the right up to length l. +func RightPadBytes(slice []byte, l int) []byte { + if l <= len(slice) { + return slice + } + + padded := make([]byte, l) + copy(padded, slice) + + return padded +} + +// LeftPadBytes zero-pads slice to the left up to length l. +func LeftPadBytes(slice []byte, l int) []byte { + if l <= len(slice) { + return slice + } + + padded := make([]byte, l) + copy(padded[l-len(slice):], slice) + + return padded +} + +// TrimLeftZeroes returns a subslice of s without leading zeroes +func TrimLeftZeroes(s []byte) []byte { + idx := 0 + for ; idx < len(s); idx++ { + if s[idx] != 0 { + break + } + } + return s[idx:] +} + +// TrimRightZeroes returns a subslice of s without trailing zeroes +func TrimRightZeroes(s []byte) []byte { + idx := len(s) + for ; idx > 0; idx-- { + if s[idx-1] != 0 { + break + } + } + return s[:idx] +} diff --git a/encoding/common/bytes_test.go b/encoding/common/bytes_test.go new file mode 100644 index 000000000..0e3ec974e --- /dev/null +++ b/encoding/common/bytes_test.go @@ -0,0 +1,126 @@ +// Copyright 2014 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package common + +import ( + "bytes" + "testing" +) + +func TestCopyBytes(t *testing.T) { + input := []byte{1, 2, 3, 4} + + v := CopyBytes(input) + if !bytes.Equal(v, []byte{1, 2, 3, 4}) { + t.Fatal("not equal after copy") + } + v[0] = 99 + if bytes.Equal(v, input) { + t.Fatal("result is not a copy") + } +} + +func TestLeftPadBytes(t *testing.T) { + val := []byte{1, 2, 3, 4} + padded := []byte{0, 0, 0, 0, 1, 2, 3, 4} + + if r := LeftPadBytes(val, 8); !bytes.Equal(r, padded) { + t.Fatalf("LeftPadBytes(%v, 8) == %v", val, r) + } + if r := LeftPadBytes(val, 2); !bytes.Equal(r, val) { + t.Fatalf("LeftPadBytes(%v, 2) == %v", val, r) + } +} + +func TestRightPadBytes(t *testing.T) { + val := []byte{1, 2, 3, 4} + padded := []byte{1, 2, 3, 4, 0, 0, 0, 0} + + if r := RightPadBytes(val, 8); !bytes.Equal(r, padded) { + t.Fatalf("RightPadBytes(%v, 8) == %v", val, r) + } + if r := RightPadBytes(val, 2); !bytes.Equal(r, val) { + t.Fatalf("RightPadBytes(%v, 2) == %v", val, r) + } +} + +func TestFromHex(t *testing.T) { + input := "0x01" + expected := []byte{1} + result := FromHex(input) + if !bytes.Equal(expected, result) { + t.Errorf("Expected %x got %x", expected, result) + } +} + +func TestIsHex(t *testing.T) { + tests := []struct { + input string + ok bool + }{ + {"", true}, + {"0", false}, + {"00", true}, + {"a9e67e", true}, + {"A9E67E", true}, + {"0xa9e67e", false}, + {"a9e67e001", false}, + {"0xHELLO_MY_NAME_IS_STEVEN_@#$^&*", false}, + } + for _, test := range tests { + if ok := isHex(test.input); ok != test.ok { + t.Errorf("isHex(%q) = %v, want %v", test.input, ok, test.ok) + } + } +} + +func TestFromHexOddLength(t *testing.T) { + input := "0x1" + expected := []byte{1} + result := FromHex(input) + if !bytes.Equal(expected, result) { + t.Errorf("Expected %x got %x", expected, result) + } +} + +func TestNoPrefixShortHexOddLength(t *testing.T) { + input := "1" + expected := []byte{1} + result := FromHex(input) + if !bytes.Equal(expected, result) { + t.Errorf("Expected %x got %x", expected, result) + } +} + +func TestTrimRightZeroes(t *testing.T) { + tests := []struct { + arr []byte + exp []byte + }{ + {FromHex("0x00ffff00ff0000"), FromHex("0x00ffff00ff")}, + {FromHex("0x00000000000000"), []byte{}}, + {FromHex("0xff"), FromHex("0xff")}, + {[]byte{}, []byte{}}, + {FromHex("0x00ffffffffffff"), FromHex("0x00ffffffffffff")}, + } + for i, test := range tests { + got := TrimRightZeroes(test.arr) + if !bytes.Equal(got, test.exp) { + t.Errorf("test %d, got %x exp %x", i, got, test.exp) + } + } +} diff --git a/encoding/common/compiler/helpers.go b/encoding/common/compiler/helpers.go new file mode 100644 index 000000000..063fc1081 --- /dev/null +++ b/encoding/common/compiler/helpers.go @@ -0,0 +1,45 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Package compiler wraps the Solidity and Vyper compiler executables (solc; vyper). +package compiler + +// Contract contains information about a compiled contract, alongside its code and runtime code. +type Contract struct { + Code string `json:"code"` + RuntimeCode string `json:"runtime-code"` + Info ContractInfo `json:"info"` + Hashes map[string]string `json:"hashes"` +} + +// ContractInfo contains information about a compiled contract, including access +// to the ABI definition, source mapping, user and developer docs, and metadata. +// +// Depending on the source, language version, compiler version, and compiler +// options will provide information about how the contract was compiled. +type ContractInfo struct { + Source string `json:"source"` + Language string `json:"language"` + LanguageVersion string `json:"languageVersion"` + CompilerVersion string `json:"compilerVersion"` + CompilerOptions string `json:"compilerOptions"` + SrcMap interface{} `json:"srcMap"` + SrcMapRuntime string `json:"srcMapRuntime"` + AbiDefinition interface{} `json:"abiDefinition"` + UserDoc interface{} `json:"userDoc"` + DeveloperDoc interface{} `json:"developerDoc"` + Metadata string `json:"metadata"` +} diff --git a/encoding/common/compiler/solidity.go b/encoding/common/compiler/solidity.go new file mode 100644 index 000000000..ad8a44aa0 --- /dev/null +++ b/encoding/common/compiler/solidity.go @@ -0,0 +1,129 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Package compiler wraps the ABI compilation outputs. +package compiler + +import ( + "encoding/json" + "fmt" +) + +// --combined-output format +type solcOutput struct { + Contracts map[string]struct { + BinRuntime string `json:"bin-runtime"` + SrcMapRuntime string `json:"srcmap-runtime"` + Bin, SrcMap, Abi, Devdoc, Userdoc, Metadata string + Hashes map[string]string + } + Version string +} + +// solidity v.0.8 changes the way ABI, Devdoc and Userdoc are serialized +type solcOutputV8 struct { + Contracts map[string]struct { + BinRuntime string `json:"bin-runtime"` + SrcMapRuntime string `json:"srcmap-runtime"` + Bin, SrcMap, Metadata string + Abi interface{} + Devdoc interface{} + Userdoc interface{} + Hashes map[string]string + } + Version string +} + +// ParseCombinedJSON takes the direct output of a solc --combined-output run and +// parses it into a map of string contract name to Contract structs. The +// provided source, language and compiler version, and compiler options are all +// passed through into the Contract structs. +// +// The solc output is expected to contain ABI, source mapping, user docs, and dev docs. +// +// Returns an error if the JSON is malformed or missing data, or if the JSON +// embedded within the JSON is malformed. +func ParseCombinedJSON(combinedJSON []byte, source string, languageVersion string, compilerVersion string, compilerOptions string) (map[string]*Contract, error) { + var output solcOutput + if err := json.Unmarshal(combinedJSON, &output); err != nil { + // Try to parse the output with the new solidity v.0.8.0 rules + return parseCombinedJSONV8(combinedJSON, source, languageVersion, compilerVersion, compilerOptions) + } + // Compilation succeeded, assemble and return the contracts. + contracts := make(map[string]*Contract) + for name, info := range output.Contracts { + // Parse the individual compilation results. + var abi interface{} + if err := json.Unmarshal([]byte(info.Abi), &abi); err != nil { + return nil, fmt.Errorf("solc: error reading abi definition (%v)", err) + } + var userdoc, devdoc interface{} + json.Unmarshal([]byte(info.Userdoc), &userdoc) + json.Unmarshal([]byte(info.Devdoc), &devdoc) + + contracts[name] = &Contract{ + Code: "0x" + info.Bin, + RuntimeCode: "0x" + info.BinRuntime, + Hashes: info.Hashes, + Info: ContractInfo{ + Source: source, + Language: "Solidity", + LanguageVersion: languageVersion, + CompilerVersion: compilerVersion, + CompilerOptions: compilerOptions, + SrcMap: info.SrcMap, + SrcMapRuntime: info.SrcMapRuntime, + AbiDefinition: abi, + UserDoc: userdoc, + DeveloperDoc: devdoc, + Metadata: info.Metadata, + }, + } + } + return contracts, nil +} + +// parseCombinedJSONV8 parses the direct output of solc --combined-output +// and parses it using the rules from solidity v.0.8.0 and later. +func parseCombinedJSONV8(combinedJSON []byte, source string, languageVersion string, compilerVersion string, compilerOptions string) (map[string]*Contract, error) { + var output solcOutputV8 + if err := json.Unmarshal(combinedJSON, &output); err != nil { + return nil, err + } + // Compilation succeeded, assemble and return the contracts. + contracts := make(map[string]*Contract) + for name, info := range output.Contracts { + contracts[name] = &Contract{ + Code: "0x" + info.Bin, + RuntimeCode: "0x" + info.BinRuntime, + Hashes: info.Hashes, + Info: ContractInfo{ + Source: source, + Language: "Solidity", + LanguageVersion: languageVersion, + CompilerVersion: compilerVersion, + CompilerOptions: compilerOptions, + SrcMap: info.SrcMap, + SrcMapRuntime: info.SrcMapRuntime, + AbiDefinition: info.Abi, + UserDoc: info.Userdoc, + DeveloperDoc: info.Devdoc, + Metadata: info.Metadata, + }, + } + } + return contracts, nil +} diff --git a/encoding/common/debug.go b/encoding/common/debug.go new file mode 100644 index 000000000..28c52b4a9 --- /dev/null +++ b/encoding/common/debug.go @@ -0,0 +1,52 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package common + +import ( + "fmt" + "os" + "runtime" + "runtime/debug" + "strings" +) + +// Report gives off a warning requesting the user to submit an issue to the github tracker. +func Report(extra ...interface{}) { + fmt.Fprintln(os.Stderr, "You've encountered a sought after, hard to reproduce bug. Please report this to the developers <3 https://github.com/ethereum/go-ethereum/issues") + fmt.Fprintln(os.Stderr, extra...) + + _, file, line, _ := runtime.Caller(1) + fmt.Fprintf(os.Stderr, "%v:%v\n", file, line) + + debug.PrintStack() + + fmt.Fprintln(os.Stderr, "#### BUG! PLEASE REPORT ####") +} + +// PrintDeprecationWarning prints the given string in a box using fmt.Println. +func PrintDeprecationWarning(str string) { + line := strings.Repeat("#", len(str)+4) + emptyLine := strings.Repeat(" ", len(str)) + fmt.Printf(` +%s +# %s # +# %s # +# %s # +%s + +`, line, emptyLine, str, emptyLine, line) +} diff --git a/encoding/common/fdlimit/fdlimit_bsd.go b/encoding/common/fdlimit/fdlimit_bsd.go new file mode 100644 index 000000000..a3a6902c0 --- /dev/null +++ b/encoding/common/fdlimit/fdlimit_bsd.go @@ -0,0 +1,68 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +//go:build freebsd || dragonfly +// +build freebsd dragonfly + +package fdlimit + +import "syscall" + +// This file is largely identical to fdlimit_unix.go, +// but Rlimit fields have type int64 on *BSD so it needs +// an extra conversion. + +// Raise tries to maximize the file descriptor allowance of this process +// to the maximum hard-limit allowed by the OS. +func Raise(max uint64) (uint64, error) { + // Get the current limit + var limit syscall.Rlimit + if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil { + return 0, err + } + // Try to update the limit to the max allowance + limit.Cur = limit.Max + if limit.Cur > int64(max) { + limit.Cur = int64(max) + } + if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil { + return 0, err + } + if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil { + return 0, err + } + return uint64(limit.Cur), nil +} + +// Current retrieves the number of file descriptors allowed to be opened by this +// process. +func Current() (int, error) { + var limit syscall.Rlimit + if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil { + return 0, err + } + return int(limit.Cur), nil +} + +// Maximum retrieves the maximum number of file descriptors this process is +// allowed to request for itself. +func Maximum() (int, error) { + var limit syscall.Rlimit + if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil { + return 0, err + } + return int(limit.Max), nil +} diff --git a/encoding/common/fdlimit/fdlimit_darwin.go b/encoding/common/fdlimit/fdlimit_darwin.go new file mode 100644 index 000000000..6b26fa00f --- /dev/null +++ b/encoding/common/fdlimit/fdlimit_darwin.go @@ -0,0 +1,71 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package fdlimit + +import "syscall" + +// hardlimit is the number of file descriptors allowed at max by the kernel. +const hardlimit = 10240 + +// Raise tries to maximize the file descriptor allowance of this process +// to the maximum hard-limit allowed by the OS. +// Returns the size it was set to (may differ from the desired 'max') +func Raise(max uint64) (uint64, error) { + // Get the current limit + var limit syscall.Rlimit + if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil { + return 0, err + } + // Try to update the limit to the max allowance + limit.Cur = limit.Max + if limit.Cur > max { + limit.Cur = max + } + if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil { + return 0, err + } + // MacOS can silently apply further caps, so retrieve the actually set limit + if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil { + return 0, err + } + return limit.Cur, nil +} + +// Current retrieves the number of file descriptors allowed to be opened by this +// process. +func Current() (int, error) { + var limit syscall.Rlimit + if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil { + return 0, err + } + return int(limit.Cur), nil +} + +// Maximum retrieves the maximum number of file descriptors this process is +// allowed to request for itself. +func Maximum() (int, error) { + // Retrieve the maximum allowed by dynamic OS limits + var limit syscall.Rlimit + if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil { + return 0, err + } + // Cap it to OPEN_MAX (10240) because macos is a special snowflake + if limit.Max > hardlimit { + limit.Max = hardlimit + } + return int(limit.Max), nil +} diff --git a/encoding/common/fdlimit/fdlimit_test.go b/encoding/common/fdlimit/fdlimit_test.go new file mode 100644 index 000000000..21362b846 --- /dev/null +++ b/encoding/common/fdlimit/fdlimit_test.go @@ -0,0 +1,45 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package fdlimit + +import ( + "fmt" + "testing" +) + +// TestFileDescriptorLimits simply tests whether the file descriptor allowance +// per this process can be retrieved. +func TestFileDescriptorLimits(t *testing.T) { + target := 4096 + hardlimit, err := Maximum() + if err != nil { + t.Fatal(err) + } + if hardlimit < target { + t.Skip(fmt.Sprintf("system limit is less than desired test target: %d < %d", hardlimit, target)) + } + + if limit, err := Current(); err != nil || limit <= 0 { + t.Fatalf("failed to retrieve file descriptor limit (%d): %v", limit, err) + } + if _, err := Raise(uint64(target)); err != nil { + t.Fatalf("failed to raise file allowance") + } + if limit, err := Current(); err != nil || limit < target { + t.Fatalf("failed to retrieve raised descriptor limit (have %v, want %v): %v", limit, target, err) + } +} diff --git a/encoding/common/fdlimit/fdlimit_unix.go b/encoding/common/fdlimit/fdlimit_unix.go new file mode 100644 index 000000000..a1f388ebb --- /dev/null +++ b/encoding/common/fdlimit/fdlimit_unix.go @@ -0,0 +1,66 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +//go:build linux || netbsd || openbsd || solaris +// +build linux netbsd openbsd solaris + +package fdlimit + +import "syscall" + +// Raise tries to maximize the file descriptor allowance of this process +// to the maximum hard-limit allowed by the OS. +// Returns the size it was set to (may differ from the desired 'max') +func Raise(max uint64) (uint64, error) { + // Get the current limit + var limit syscall.Rlimit + if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil { + return 0, err + } + // Try to update the limit to the max allowance + limit.Cur = limit.Max + if limit.Cur > max { + limit.Cur = max + } + if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil { + return 0, err + } + // MacOS can silently apply further caps, so retrieve the actually set limit + if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil { + return 0, err + } + return limit.Cur, nil +} + +// Current retrieves the number of file descriptors allowed to be opened by this +// process. +func Current() (int, error) { + var limit syscall.Rlimit + if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil { + return 0, err + } + return int(limit.Cur), nil +} + +// Maximum retrieves the maximum number of file descriptors this process is +// allowed to request for itself. +func Maximum() (int, error) { + var limit syscall.Rlimit + if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil { + return 0, err + } + return int(limit.Max), nil +} diff --git a/encoding/common/fdlimit/fdlimit_windows.go b/encoding/common/fdlimit/fdlimit_windows.go new file mode 100644 index 000000000..f47215366 --- /dev/null +++ b/encoding/common/fdlimit/fdlimit_windows.go @@ -0,0 +1,50 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package fdlimit + +import "fmt" + +// hardlimit is the number of file descriptors allowed at max by the kernel. +const hardlimit = 16384 + +// Raise tries to maximize the file descriptor allowance of this process +// to the maximum hard-limit allowed by the OS. +func Raise(max uint64) (uint64, error) { + // This method is NOP by design: + // * Linux/Darwin counterparts need to manually increase per process limits + // * On Windows Go uses the CreateFile API, which is limited to 16K files, non + // changeable from within a running process + // This way we can always "request" raising the limits, which will either have + // or not have effect based on the platform we're running on. + if max > hardlimit { + return hardlimit, fmt.Errorf("file descriptor limit (%d) reached", hardlimit) + } + return max, nil +} + +// Current retrieves the number of file descriptors allowed to be opened by this +// process. +func Current() (int, error) { + // Please see Raise for the reason why we use hard coded 16K as the limit + return hardlimit, nil +} + +// Maximum retrieves the maximum number of file descriptors this process is +// allowed to request for itself. +func Maximum() (int, error) { + return Current() +} diff --git a/encoding/common/format.go b/encoding/common/format.go new file mode 100644 index 000000000..6fc21af71 --- /dev/null +++ b/encoding/common/format.go @@ -0,0 +1,82 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package common + +import ( + "fmt" + "regexp" + "strings" + "time" +) + +// PrettyDuration is a pretty printed version of a time.Duration value that cuts +// the unnecessary precision off from the formatted textual representation. +type PrettyDuration time.Duration + +var prettyDurationRe = regexp.MustCompile(`\.[0-9]+`) + +// String implements the Stringer interface, allowing pretty printing of duration +// values rounded to three decimals. +func (d PrettyDuration) String() string { + label := fmt.Sprintf("%v", time.Duration(d)) + if match := prettyDurationRe.FindString(label); len(match) > 4 { + label = strings.Replace(label, match, match[:4], 1) + } + return label +} + +// PrettyAge is a pretty printed version of a time.Duration value that rounds +// the values up to a single most significant unit, days/weeks/years included. +type PrettyAge time.Time + +// ageUnits is a list of units the age pretty printing uses. +var ageUnits = []struct { + Size time.Duration + Symbol string +}{ + {12 * 30 * 24 * time.Hour, "y"}, + {30 * 24 * time.Hour, "mo"}, + {7 * 24 * time.Hour, "w"}, + {24 * time.Hour, "d"}, + {time.Hour, "h"}, + {time.Minute, "m"}, + {time.Second, "s"}, +} + +// String implements the Stringer interface, allowing pretty printing of duration +// values rounded to the most significant time unit. +func (t PrettyAge) String() string { + // Calculate the time difference and handle the 0 cornercase + diff := time.Since(time.Time(t)) + if diff < time.Second { + return "0" + } + // Accumulate a precision of 3 components before returning + result, prec := "", 0 + + for _, unit := range ageUnits { + if diff > unit.Size { + result = fmt.Sprintf("%s%d%s", result, diff/unit.Size, unit.Symbol) + diff %= unit.Size + + if prec += 1; prec >= 3 { + break + } + } + } + return result +} diff --git a/encoding/common/hexutil/hexutil.go b/encoding/common/hexutil/hexutil.go new file mode 100644 index 000000000..e0241f5f2 --- /dev/null +++ b/encoding/common/hexutil/hexutil.go @@ -0,0 +1,241 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +/* +Package hexutil implements hex encoding with 0x prefix. +This encoding is used by the Ethereum RPC API to transport binary data in JSON payloads. + +Encoding Rules + +All hex data must have prefix "0x". + +For byte slices, the hex data must be of even length. An empty byte slice +encodes as "0x". + +Integers are encoded using the least amount of digits (no leading zero digits). Their +encoding may be of uneven length. The number zero encodes as "0x0". +*/ +package hexutil + +import ( + "encoding/hex" + "fmt" + "math/big" + "strconv" +) + +const uintBits = 32 << (uint64(^uint(0)) >> 63) + +// Errors +var ( + ErrEmptyString = &decError{"empty hex string"} + ErrSyntax = &decError{"invalid hex string"} + ErrMissingPrefix = &decError{"hex string without 0x prefix"} + ErrOddLength = &decError{"hex string of odd length"} + ErrEmptyNumber = &decError{"hex string \"0x\""} + ErrLeadingZero = &decError{"hex number with leading zero digits"} + ErrUint64Range = &decError{"hex number > 64 bits"} + ErrUintRange = &decError{fmt.Sprintf("hex number > %d bits", uintBits)} + ErrBig256Range = &decError{"hex number > 256 bits"} +) + +type decError struct{ msg string } + +func (err decError) Error() string { return err.msg } + +// Decode decodes a hex string with 0x prefix. +func Decode(input string) ([]byte, error) { + if len(input) == 0 { + return nil, ErrEmptyString + } + if !has0xPrefix(input) { + return nil, ErrMissingPrefix + } + b, err := hex.DecodeString(input[2:]) + if err != nil { + err = mapError(err) + } + return b, err +} + +// MustDecode decodes a hex string with 0x prefix. It panics for invalid input. +func MustDecode(input string) []byte { + dec, err := Decode(input) + if err != nil { + panic(err) + } + return dec +} + +// Encode encodes b as a hex string with 0x prefix. +func Encode(b []byte) string { + enc := make([]byte, len(b)*2+2) + copy(enc, "0x") + hex.Encode(enc[2:], b) + return string(enc) +} + +// DecodeUint64 decodes a hex string with 0x prefix as a quantity. +func DecodeUint64(input string) (uint64, error) { + raw, err := checkNumber(input) + if err != nil { + return 0, err + } + dec, err := strconv.ParseUint(raw, 16, 64) + if err != nil { + err = mapError(err) + } + return dec, err +} + +// MustDecodeUint64 decodes a hex string with 0x prefix as a quantity. +// It panics for invalid input. +func MustDecodeUint64(input string) uint64 { + dec, err := DecodeUint64(input) + if err != nil { + panic(err) + } + return dec +} + +// EncodeUint64 encodes i as a hex string with 0x prefix. +func EncodeUint64(i uint64) string { + enc := make([]byte, 2, 10) + copy(enc, "0x") + return string(strconv.AppendUint(enc, i, 16)) +} + +var bigWordNibbles int + +func init() { + // This is a weird way to compute the number of nibbles required for big.Word. + // The usual way would be to use constant arithmetic but go vet can't handle that. + b, _ := new(big.Int).SetString("FFFFFFFFFF", 16) + switch len(b.Bits()) { + case 1: + bigWordNibbles = 16 + case 2: + bigWordNibbles = 8 + default: + panic("weird big.Word size") + } +} + +// DecodeBig decodes a hex string with 0x prefix as a quantity. +// Numbers larger than 256 bits are not accepted. +func DecodeBig(input string) (*big.Int, error) { + raw, err := checkNumber(input) + if err != nil { + return nil, err + } + if len(raw) > 64 { + return nil, ErrBig256Range + } + words := make([]big.Word, len(raw)/bigWordNibbles+1) + end := len(raw) + for i := range words { + start := end - bigWordNibbles + if start < 0 { + start = 0 + } + for ri := start; ri < end; ri++ { + nib := decodeNibble(raw[ri]) + if nib == badNibble { + return nil, ErrSyntax + } + words[i] *= 16 + words[i] += big.Word(nib) + } + end = start + } + dec := new(big.Int).SetBits(words) + return dec, nil +} + +// MustDecodeBig decodes a hex string with 0x prefix as a quantity. +// It panics for invalid input. +func MustDecodeBig(input string) *big.Int { + dec, err := DecodeBig(input) + if err != nil { + panic(err) + } + return dec +} + +// EncodeBig encodes bigint as a hex string with 0x prefix. +func EncodeBig(bigint *big.Int) string { + if sign := bigint.Sign(); sign == 0 { + return "0x0" + } else if sign > 0 { + return "0x" + bigint.Text(16) + } else { + return "-0x" + bigint.Text(16)[1:] + } +} + +func has0xPrefix(input string) bool { + return len(input) >= 2 && input[0] == '0' && (input[1] == 'x' || input[1] == 'X') +} + +func checkNumber(input string) (raw string, err error) { + if len(input) == 0 { + return "", ErrEmptyString + } + if !has0xPrefix(input) { + return "", ErrMissingPrefix + } + input = input[2:] + if len(input) == 0 { + return "", ErrEmptyNumber + } + if len(input) > 1 && input[0] == '0' { + return "", ErrLeadingZero + } + return input, nil +} + +const badNibble = ^uint64(0) + +func decodeNibble(in byte) uint64 { + switch { + case in >= '0' && in <= '9': + return uint64(in - '0') + case in >= 'A' && in <= 'F': + return uint64(in - 'A' + 10) + case in >= 'a' && in <= 'f': + return uint64(in - 'a' + 10) + default: + return badNibble + } +} + +func mapError(err error) error { + if err, ok := err.(*strconv.NumError); ok { + switch err.Err { + case strconv.ErrRange: + return ErrUint64Range + case strconv.ErrSyntax: + return ErrSyntax + } + } + if _, ok := err.(hex.InvalidByteError); ok { + return ErrSyntax + } + if err == hex.ErrLength { + return ErrOddLength + } + return err +} diff --git a/encoding/common/hexutil/hexutil_test.go b/encoding/common/hexutil/hexutil_test.go new file mode 100644 index 000000000..f2b800d82 --- /dev/null +++ b/encoding/common/hexutil/hexutil_test.go @@ -0,0 +1,215 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package hexutil + +import ( + "bytes" + "math/big" + "testing" +) + +type marshalTest struct { + input interface{} + want string +} + +type unmarshalTest struct { + input string + want interface{} + wantErr error // if set, decoding must fail on any platform + wantErr32bit error // if set, decoding must fail on 32bit platforms (used for Uint tests) +} + +var ( + encodeBytesTests = []marshalTest{ + {[]byte{}, "0x"}, + {[]byte{0}, "0x00"}, + {[]byte{0, 0, 1, 2}, "0x00000102"}, + } + + encodeBigTests = []marshalTest{ + {referenceBig("0"), "0x0"}, + {referenceBig("1"), "0x1"}, + {referenceBig("ff"), "0xff"}, + {referenceBig("112233445566778899aabbccddeeff"), "0x112233445566778899aabbccddeeff"}, + {referenceBig("80a7f2c1bcc396c00"), "0x80a7f2c1bcc396c00"}, + {referenceBig("-80a7f2c1bcc396c00"), "-0x80a7f2c1bcc396c00"}, + } + + encodeUint64Tests = []marshalTest{ + {uint64(0), "0x0"}, + {uint64(1), "0x1"}, + {uint64(0xff), "0xff"}, + {uint64(0x1122334455667788), "0x1122334455667788"}, + } + + encodeUintTests = []marshalTest{ + {uint(0), "0x0"}, + {uint(1), "0x1"}, + {uint(0xff), "0xff"}, + {uint(0x11223344), "0x11223344"}, + } + + decodeBytesTests = []unmarshalTest{ + // invalid + {input: ``, wantErr: ErrEmptyString}, + {input: `0`, wantErr: ErrMissingPrefix}, + {input: `0x0`, wantErr: ErrOddLength}, + {input: `0x023`, wantErr: ErrOddLength}, + {input: `0xxx`, wantErr: ErrSyntax}, + {input: `0x01zz01`, wantErr: ErrSyntax}, + // valid + {input: `0x`, want: []byte{}}, + {input: `0X`, want: []byte{}}, + {input: `0x02`, want: []byte{0x02}}, + {input: `0X02`, want: []byte{0x02}}, + {input: `0xffffffffff`, want: []byte{0xff, 0xff, 0xff, 0xff, 0xff}}, + { + input: `0xffffffffffffffffffffffffffffffffffff`, + want: []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, + }, + } + + decodeBigTests = []unmarshalTest{ + // invalid + {input: `0`, wantErr: ErrMissingPrefix}, + {input: `0x`, wantErr: ErrEmptyNumber}, + {input: `0x01`, wantErr: ErrLeadingZero}, + {input: `0xx`, wantErr: ErrSyntax}, + {input: `0x1zz01`, wantErr: ErrSyntax}, + { + input: `0x10000000000000000000000000000000000000000000000000000000000000000`, + wantErr: ErrBig256Range, + }, + // valid + {input: `0x0`, want: big.NewInt(0)}, + {input: `0x2`, want: big.NewInt(0x2)}, + {input: `0x2F2`, want: big.NewInt(0x2f2)}, + {input: `0X2F2`, want: big.NewInt(0x2f2)}, + {input: `0x1122aaff`, want: big.NewInt(0x1122aaff)}, + {input: `0xbBb`, want: big.NewInt(0xbbb)}, + {input: `0xfffffffff`, want: big.NewInt(0xfffffffff)}, + { + input: `0x112233445566778899aabbccddeeff`, + want: referenceBig("112233445566778899aabbccddeeff"), + }, + { + input: `0xffffffffffffffffffffffffffffffffffff`, + want: referenceBig("ffffffffffffffffffffffffffffffffffff"), + }, + { + input: `0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff`, + want: referenceBig("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), + }, + } + + decodeUint64Tests = []unmarshalTest{ + // invalid + {input: `0`, wantErr: ErrMissingPrefix}, + {input: `0x`, wantErr: ErrEmptyNumber}, + {input: `0x01`, wantErr: ErrLeadingZero}, + {input: `0xfffffffffffffffff`, wantErr: ErrUint64Range}, + {input: `0xx`, wantErr: ErrSyntax}, + {input: `0x1zz01`, wantErr: ErrSyntax}, + // valid + {input: `0x0`, want: uint64(0)}, + {input: `0x2`, want: uint64(0x2)}, + {input: `0x2F2`, want: uint64(0x2f2)}, + {input: `0X2F2`, want: uint64(0x2f2)}, + {input: `0x1122aaff`, want: uint64(0x1122aaff)}, + {input: `0xbbb`, want: uint64(0xbbb)}, + {input: `0xffffffffffffffff`, want: uint64(0xffffffffffffffff)}, + } +) + +func TestEncode(t *testing.T) { + for _, test := range encodeBytesTests { + enc := Encode(test.input.([]byte)) + if enc != test.want { + t.Errorf("input %x: wrong encoding %s", test.input, enc) + } + } +} + +func TestDecode(t *testing.T) { + for _, test := range decodeBytesTests { + dec, err := Decode(test.input) + if !checkError(t, test.input, err, test.wantErr) { + continue + } + if !bytes.Equal(test.want.([]byte), dec) { + t.Errorf("input %s: value mismatch: got %x, want %x", test.input, dec, test.want) + continue + } + } +} + +func TestEncodeBig(t *testing.T) { + for _, test := range encodeBigTests { + enc := EncodeBig(test.input.(*big.Int)) + if enc != test.want { + t.Errorf("input %x: wrong encoding %s", test.input, enc) + } + } +} + +func TestDecodeBig(t *testing.T) { + for _, test := range decodeBigTests { + dec, err := DecodeBig(test.input) + if !checkError(t, test.input, err, test.wantErr) { + continue + } + if dec.Cmp(test.want.(*big.Int)) != 0 { + t.Errorf("input %s: value mismatch: got %x, want %x", test.input, dec, test.want) + continue + } + } +} + +func TestEncodeUint64(t *testing.T) { + for _, test := range encodeUint64Tests { + enc := EncodeUint64(test.input.(uint64)) + if enc != test.want { + t.Errorf("input %x: wrong encoding %s", test.input, enc) + } + } +} + +func TestDecodeUint64(t *testing.T) { + for _, test := range decodeUint64Tests { + dec, err := DecodeUint64(test.input) + if !checkError(t, test.input, err, test.wantErr) { + continue + } + if dec != test.want.(uint64) { + t.Errorf("input %s: value mismatch: got %x, want %x", test.input, dec, test.want) + continue + } + } +} + +func BenchmarkEncodeBig(b *testing.B) { + for _, bench := range encodeBigTests { + b.Run(bench.want, func(b *testing.B) { + b.ReportAllocs() + bigint := bench.input.(*big.Int) + for i := 0; i < b.N; i++ { + EncodeBig(bigint) + } + }) + } +} diff --git a/encoding/common/hexutil/json.go b/encoding/common/hexutil/json.go new file mode 100644 index 000000000..50db20811 --- /dev/null +++ b/encoding/common/hexutil/json.go @@ -0,0 +1,376 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package hexutil + +import ( + "encoding/hex" + "encoding/json" + "fmt" + "math/big" + "reflect" + "strconv" +) + +var ( + bytesT = reflect.TypeOf(Bytes(nil)) + bigT = reflect.TypeOf((*Big)(nil)) + uintT = reflect.TypeOf(Uint(0)) + uint64T = reflect.TypeOf(Uint64(0)) +) + +// Bytes marshals/unmarshals as a JSON string with 0x prefix. +// The empty slice marshals as "0x". +type Bytes []byte + +// MarshalText implements encoding.TextMarshaler +func (b Bytes) MarshalText() ([]byte, error) { + result := make([]byte, len(b)*2+2) + copy(result, `0x`) + hex.Encode(result[2:], b) + return result, nil +} + +// UnmarshalJSON implements json.Unmarshaler. +func (b *Bytes) UnmarshalJSON(input []byte) error { + if !isString(input) { + return errNonString(bytesT) + } + return wrapTypeError(b.UnmarshalText(input[1:len(input)-1]), bytesT) +} + +// UnmarshalText implements encoding.TextUnmarshaler. +func (b *Bytes) UnmarshalText(input []byte) error { + raw, err := checkText(input, true) + if err != nil { + return err + } + dec := make([]byte, len(raw)/2) + if _, err = hex.Decode(dec, raw); err != nil { + err = mapError(err) + } else { + *b = dec + } + return err +} + +// String returns the hex encoding of b. +func (b Bytes) String() string { + return Encode(b) +} + +// ImplementsGraphQLType returns true if Bytes implements the specified GraphQL type. +func (b Bytes) ImplementsGraphQLType(name string) bool { return name == "Bytes" } + +// UnmarshalGraphQL unmarshals the provided GraphQL query data. +func (b *Bytes) UnmarshalGraphQL(input interface{}) error { + var err error + switch input := input.(type) { + case string: + data, err := Decode(input) + if err != nil { + return err + } + *b = data + default: + err = fmt.Errorf("unexpected type %T for Bytes", input) + } + return err +} + +// UnmarshalFixedJSON decodes the input as a string with 0x prefix. The length of out +// determines the required input length. This function is commonly used to implement the +// UnmarshalJSON method for fixed-size types. +func UnmarshalFixedJSON(typ reflect.Type, input, out []byte) error { + if !isString(input) { + return errNonString(typ) + } + return wrapTypeError(UnmarshalFixedText(typ.String(), input[1:len(input)-1], out), typ) +} + +// UnmarshalFixedText decodes the input as a string with 0x prefix. The length of out +// determines the required input length. This function is commonly used to implement the +// UnmarshalText method for fixed-size types. +func UnmarshalFixedText(typname string, input, out []byte) error { + raw, err := checkText(input, true) + if err != nil { + return err + } + if len(raw)/2 != len(out) { + return fmt.Errorf("hex string has length %d, want %d for %s", len(raw), len(out)*2, typname) + } + // Pre-verify syntax before modifying out. + for _, b := range raw { + if decodeNibble(b) == badNibble { + return ErrSyntax + } + } + hex.Decode(out, raw) + return nil +} + +// UnmarshalFixedUnprefixedText decodes the input as a string with optional 0x prefix. The +// length of out determines the required input length. This function is commonly used to +// implement the UnmarshalText method for fixed-size types. +func UnmarshalFixedUnprefixedText(typname string, input, out []byte) error { + raw, err := checkText(input, false) + if err != nil { + return err + } + if len(raw)/2 != len(out) { + return fmt.Errorf("hex string has length %d, want %d for %s", len(raw), len(out)*2, typname) + } + // Pre-verify syntax before modifying out. + for _, b := range raw { + if decodeNibble(b) == badNibble { + return ErrSyntax + } + } + hex.Decode(out, raw) + return nil +} + +// Big marshals/unmarshals as a JSON string with 0x prefix. +// The zero value marshals as "0x0". +// +// Negative integers are not supported at this time. Attempting to marshal them will +// return an error. Values larger than 256bits are rejected by Unmarshal but will be +// marshaled without error. +type Big big.Int + +// MarshalText implements encoding.TextMarshaler +func (b Big) MarshalText() ([]byte, error) { + return []byte(EncodeBig((*big.Int)(&b))), nil +} + +// UnmarshalJSON implements json.Unmarshaler. +func (b *Big) UnmarshalJSON(input []byte) error { + if !isString(input) { + return errNonString(bigT) + } + return wrapTypeError(b.UnmarshalText(input[1:len(input)-1]), bigT) +} + +// UnmarshalText implements encoding.TextUnmarshaler +func (b *Big) UnmarshalText(input []byte) error { + raw, err := checkNumberText(input) + if err != nil { + return err + } + if len(raw) > 64 { + return ErrBig256Range + } + words := make([]big.Word, len(raw)/bigWordNibbles+1) + end := len(raw) + for i := range words { + start := end - bigWordNibbles + if start < 0 { + start = 0 + } + for ri := start; ri < end; ri++ { + nib := decodeNibble(raw[ri]) + if nib == badNibble { + return ErrSyntax + } + words[i] *= 16 + words[i] += big.Word(nib) + } + end = start + } + var dec big.Int + dec.SetBits(words) + *b = (Big)(dec) + return nil +} + +// ToInt converts b to a big.Int. +func (b *Big) ToInt() *big.Int { + return (*big.Int)(b) +} + +// String returns the hex encoding of b. +func (b *Big) String() string { + return EncodeBig(b.ToInt()) +} + +// ImplementsGraphQLType returns true if Big implements the provided GraphQL type. +func (b Big) ImplementsGraphQLType(name string) bool { return name == "BigInt" } + +// UnmarshalGraphQL unmarshals the provided GraphQL query data. +func (b *Big) UnmarshalGraphQL(input interface{}) error { + var err error + switch input := input.(type) { + case string: + return b.UnmarshalText([]byte(input)) + case int32: + var num big.Int + num.SetInt64(int64(input)) + *b = Big(num) + default: + err = fmt.Errorf("unexpected type %T for BigInt", input) + } + return err +} + +// Uint64 marshals/unmarshals as a JSON string with 0x prefix. +// The zero value marshals as "0x0". +type Uint64 uint64 + +// MarshalText implements encoding.TextMarshaler. +func (b Uint64) MarshalText() ([]byte, error) { + buf := make([]byte, 2, 10) + copy(buf, `0x`) + buf = strconv.AppendUint(buf, uint64(b), 16) + return buf, nil +} + +// UnmarshalJSON implements json.Unmarshaler. +func (b *Uint64) UnmarshalJSON(input []byte) error { + if !isString(input) { + return errNonString(uint64T) + } + return wrapTypeError(b.UnmarshalText(input[1:len(input)-1]), uint64T) +} + +// UnmarshalText implements encoding.TextUnmarshaler +func (b *Uint64) UnmarshalText(input []byte) error { + raw, err := checkNumberText(input) + if err != nil { + return err + } + if len(raw) > 16 { + return ErrUint64Range + } + var dec uint64 + for _, byte := range raw { + nib := decodeNibble(byte) + if nib == badNibble { + return ErrSyntax + } + dec *= 16 + dec += nib + } + *b = Uint64(dec) + return nil +} + +// String returns the hex encoding of b. +func (b Uint64) String() string { + return EncodeUint64(uint64(b)) +} + +// ImplementsGraphQLType returns true if Uint64 implements the provided GraphQL type. +func (b Uint64) ImplementsGraphQLType(name string) bool { return name == "Long" } + +// UnmarshalGraphQL unmarshals the provided GraphQL query data. +func (b *Uint64) UnmarshalGraphQL(input interface{}) error { + var err error + switch input := input.(type) { + case string: + return b.UnmarshalText([]byte(input)) + case int32: + *b = Uint64(input) + default: + err = fmt.Errorf("unexpected type %T for Long", input) + } + return err +} + +// Uint marshals/unmarshals as a JSON string with 0x prefix. +// The zero value marshals as "0x0". +type Uint uint + +// MarshalText implements encoding.TextMarshaler. +func (b Uint) MarshalText() ([]byte, error) { + return Uint64(b).MarshalText() +} + +// UnmarshalJSON implements json.Unmarshaler. +func (b *Uint) UnmarshalJSON(input []byte) error { + if !isString(input) { + return errNonString(uintT) + } + return wrapTypeError(b.UnmarshalText(input[1:len(input)-1]), uintT) +} + +// UnmarshalText implements encoding.TextUnmarshaler. +func (b *Uint) UnmarshalText(input []byte) error { + var u64 Uint64 + err := u64.UnmarshalText(input) + if u64 > Uint64(^uint(0)) || err == ErrUint64Range { + return ErrUintRange + } else if err != nil { + return err + } + *b = Uint(u64) + return nil +} + +// String returns the hex encoding of b. +func (b Uint) String() string { + return EncodeUint64(uint64(b)) +} + +func isString(input []byte) bool { + return len(input) >= 2 && input[0] == '"' && input[len(input)-1] == '"' +} + +func bytesHave0xPrefix(input []byte) bool { + return len(input) >= 2 && input[0] == '0' && (input[1] == 'x' || input[1] == 'X') +} + +func checkText(input []byte, wantPrefix bool) ([]byte, error) { + if len(input) == 0 { + return nil, nil // empty strings are allowed + } + if bytesHave0xPrefix(input) { + input = input[2:] + } else if wantPrefix { + return nil, ErrMissingPrefix + } + if len(input)%2 != 0 { + return nil, ErrOddLength + } + return input, nil +} + +func checkNumberText(input []byte) (raw []byte, err error) { + if len(input) == 0 { + return nil, nil // empty strings are allowed + } + if !bytesHave0xPrefix(input) { + return nil, ErrMissingPrefix + } + input = input[2:] + if len(input) == 0 { + return nil, ErrEmptyNumber + } + if len(input) > 1 && input[0] == '0' { + return nil, ErrLeadingZero + } + return input, nil +} + +func wrapTypeError(err error, typ reflect.Type) error { + if _, ok := err.(*decError); ok { + return &json.UnmarshalTypeError{Value: err.Error(), Type: typ} + } + return err +} + +func errNonString(typ reflect.Type) error { + return &json.UnmarshalTypeError{Value: "non-string", Type: typ} +} diff --git a/encoding/common/hexutil/json_example_test.go b/encoding/common/hexutil/json_example_test.go new file mode 100644 index 000000000..36a7f7254 --- /dev/null +++ b/encoding/common/hexutil/json_example_test.go @@ -0,0 +1,45 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package hexutil_test + +import ( + "encoding/json" + "fmt" + + "github.com/onflow/flow-go-sdk/encoding/common/hexutil" +) + +type MyType [5]byte + +func (v *MyType) UnmarshalText(input []byte) error { + return hexutil.UnmarshalFixedText("MyType", input, v[:]) +} + +func (v MyType) String() string { + return hexutil.Bytes(v[:]).String() +} + +func ExampleUnmarshalFixedText() { + var v1, v2 MyType + fmt.Println("v1 error:", json.Unmarshal([]byte(`"0x01"`), &v1)) + fmt.Println("v2 error:", json.Unmarshal([]byte(`"0x0101010101"`), &v2)) + fmt.Println("v2:", v2) + // Output: + // v1 error: hex string has length 2, want 10 for MyType + // v2 error: + // v2: 0x0101010101 +} diff --git a/encoding/common/hexutil/json_test.go b/encoding/common/hexutil/json_test.go new file mode 100644 index 000000000..ed7d6fad1 --- /dev/null +++ b/encoding/common/hexutil/json_test.go @@ -0,0 +1,374 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package hexutil + +import ( + "bytes" + "encoding/hex" + "encoding/json" + "errors" + "math/big" + "testing" +) + +func checkError(t *testing.T, input string, got, want error) bool { + if got == nil { + if want != nil { + t.Errorf("input %s: got no error, want %q", input, want) + return false + } + return true + } + if want == nil { + t.Errorf("input %s: unexpected error %q", input, got) + } else if got.Error() != want.Error() { + t.Errorf("input %s: got error %q, want %q", input, got, want) + } + return false +} + +func referenceBig(s string) *big.Int { + b, ok := new(big.Int).SetString(s, 16) + if !ok { + panic("invalid") + } + return b +} + +func referenceBytes(s string) []byte { + b, err := hex.DecodeString(s) + if err != nil { + panic(err) + } + return b +} + +var errJSONEOF = errors.New("unexpected end of JSON input") + +var unmarshalBytesTests = []unmarshalTest{ + // invalid encoding + {input: "", wantErr: errJSONEOF}, + {input: "null", wantErr: errNonString(bytesT)}, + {input: "10", wantErr: errNonString(bytesT)}, + {input: `"0"`, wantErr: wrapTypeError(ErrMissingPrefix, bytesT)}, + {input: `"0x0"`, wantErr: wrapTypeError(ErrOddLength, bytesT)}, + {input: `"0xxx"`, wantErr: wrapTypeError(ErrSyntax, bytesT)}, + {input: `"0x01zz01"`, wantErr: wrapTypeError(ErrSyntax, bytesT)}, + + // valid encoding + {input: `""`, want: referenceBytes("")}, + {input: `"0x"`, want: referenceBytes("")}, + {input: `"0x02"`, want: referenceBytes("02")}, + {input: `"0X02"`, want: referenceBytes("02")}, + {input: `"0xffffffffff"`, want: referenceBytes("ffffffffff")}, + { + input: `"0xffffffffffffffffffffffffffffffffffff"`, + want: referenceBytes("ffffffffffffffffffffffffffffffffffff"), + }, +} + +func TestUnmarshalBytes(t *testing.T) { + for _, test := range unmarshalBytesTests { + var v Bytes + err := json.Unmarshal([]byte(test.input), &v) + if !checkError(t, test.input, err, test.wantErr) { + continue + } + if !bytes.Equal(test.want.([]byte), v) { + t.Errorf("input %s: value mismatch: got %x, want %x", test.input, &v, test.want) + continue + } + } +} + +func BenchmarkUnmarshalBytes(b *testing.B) { + input := []byte(`"0x123456789abcdef123456789abcdef"`) + for i := 0; i < b.N; i++ { + var v Bytes + if err := v.UnmarshalJSON(input); err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalBytes(t *testing.T) { + for _, test := range encodeBytesTests { + in := test.input.([]byte) + out, err := json.Marshal(Bytes(in)) + if err != nil { + t.Errorf("%x: %v", in, err) + continue + } + if want := `"` + test.want + `"`; string(out) != want { + t.Errorf("%x: MarshalJSON output mismatch: got %q, want %q", in, out, want) + continue + } + if out := Bytes(in).String(); out != test.want { + t.Errorf("%x: String mismatch: got %q, want %q", in, out, test.want) + continue + } + } +} + +var unmarshalBigTests = []unmarshalTest{ + // invalid encoding + {input: "", wantErr: errJSONEOF}, + {input: "null", wantErr: errNonString(bigT)}, + {input: "10", wantErr: errNonString(bigT)}, + {input: `"0"`, wantErr: wrapTypeError(ErrMissingPrefix, bigT)}, + {input: `"0x"`, wantErr: wrapTypeError(ErrEmptyNumber, bigT)}, + {input: `"0x01"`, wantErr: wrapTypeError(ErrLeadingZero, bigT)}, + {input: `"0xx"`, wantErr: wrapTypeError(ErrSyntax, bigT)}, + {input: `"0x1zz01"`, wantErr: wrapTypeError(ErrSyntax, bigT)}, + { + input: `"0x10000000000000000000000000000000000000000000000000000000000000000"`, + wantErr: wrapTypeError(ErrBig256Range, bigT), + }, + + // valid encoding + {input: `""`, want: big.NewInt(0)}, + {input: `"0x0"`, want: big.NewInt(0)}, + {input: `"0x2"`, want: big.NewInt(0x2)}, + {input: `"0x2F2"`, want: big.NewInt(0x2f2)}, + {input: `"0X2F2"`, want: big.NewInt(0x2f2)}, + {input: `"0x1122aaff"`, want: big.NewInt(0x1122aaff)}, + {input: `"0xbBb"`, want: big.NewInt(0xbbb)}, + {input: `"0xfffffffff"`, want: big.NewInt(0xfffffffff)}, + { + input: `"0x112233445566778899aabbccddeeff"`, + want: referenceBig("112233445566778899aabbccddeeff"), + }, + { + input: `"0xffffffffffffffffffffffffffffffffffff"`, + want: referenceBig("ffffffffffffffffffffffffffffffffffff"), + }, + { + input: `"0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"`, + want: referenceBig("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), + }, +} + +func TestUnmarshalBig(t *testing.T) { + for _, test := range unmarshalBigTests { + var v Big + err := json.Unmarshal([]byte(test.input), &v) + if !checkError(t, test.input, err, test.wantErr) { + continue + } + if test.want != nil && test.want.(*big.Int).Cmp((*big.Int)(&v)) != 0 { + t.Errorf("input %s: value mismatch: got %x, want %x", test.input, (*big.Int)(&v), test.want) + continue + } + } +} + +func BenchmarkUnmarshalBig(b *testing.B) { + input := []byte(`"0x123456789abcdef123456789abcdef"`) + for i := 0; i < b.N; i++ { + var v Big + if err := v.UnmarshalJSON(input); err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalBig(t *testing.T) { + for _, test := range encodeBigTests { + in := test.input.(*big.Int) + out, err := json.Marshal((*Big)(in)) + if err != nil { + t.Errorf("%d: %v", in, err) + continue + } + if want := `"` + test.want + `"`; string(out) != want { + t.Errorf("%d: MarshalJSON output mismatch: got %q, want %q", in, out, want) + continue + } + if out := (*Big)(in).String(); out != test.want { + t.Errorf("%x: String mismatch: got %q, want %q", in, out, test.want) + continue + } + } +} + +var unmarshalUint64Tests = []unmarshalTest{ + // invalid encoding + {input: "", wantErr: errJSONEOF}, + {input: "null", wantErr: errNonString(uint64T)}, + {input: "10", wantErr: errNonString(uint64T)}, + {input: `"0"`, wantErr: wrapTypeError(ErrMissingPrefix, uint64T)}, + {input: `"0x"`, wantErr: wrapTypeError(ErrEmptyNumber, uint64T)}, + {input: `"0x01"`, wantErr: wrapTypeError(ErrLeadingZero, uint64T)}, + {input: `"0xfffffffffffffffff"`, wantErr: wrapTypeError(ErrUint64Range, uint64T)}, + {input: `"0xx"`, wantErr: wrapTypeError(ErrSyntax, uint64T)}, + {input: `"0x1zz01"`, wantErr: wrapTypeError(ErrSyntax, uint64T)}, + + // valid encoding + {input: `""`, want: uint64(0)}, + {input: `"0x0"`, want: uint64(0)}, + {input: `"0x2"`, want: uint64(0x2)}, + {input: `"0x2F2"`, want: uint64(0x2f2)}, + {input: `"0X2F2"`, want: uint64(0x2f2)}, + {input: `"0x1122aaff"`, want: uint64(0x1122aaff)}, + {input: `"0xbbb"`, want: uint64(0xbbb)}, + {input: `"0xffffffffffffffff"`, want: uint64(0xffffffffffffffff)}, +} + +func TestUnmarshalUint64(t *testing.T) { + for _, test := range unmarshalUint64Tests { + var v Uint64 + err := json.Unmarshal([]byte(test.input), &v) + if !checkError(t, test.input, err, test.wantErr) { + continue + } + if uint64(v) != test.want.(uint64) { + t.Errorf("input %s: value mismatch: got %d, want %d", test.input, v, test.want) + continue + } + } +} + +func BenchmarkUnmarshalUint64(b *testing.B) { + input := []byte(`"0x123456789abcdf"`) + for i := 0; i < b.N; i++ { + var v Uint64 + v.UnmarshalJSON(input) + } +} + +func TestMarshalUint64(t *testing.T) { + for _, test := range encodeUint64Tests { + in := test.input.(uint64) + out, err := json.Marshal(Uint64(in)) + if err != nil { + t.Errorf("%d: %v", in, err) + continue + } + if want := `"` + test.want + `"`; string(out) != want { + t.Errorf("%d: MarshalJSON output mismatch: got %q, want %q", in, out, want) + continue + } + if out := (Uint64)(in).String(); out != test.want { + t.Errorf("%x: String mismatch: got %q, want %q", in, out, test.want) + continue + } + } +} + +func TestMarshalUint(t *testing.T) { + for _, test := range encodeUintTests { + in := test.input.(uint) + out, err := json.Marshal(Uint(in)) + if err != nil { + t.Errorf("%d: %v", in, err) + continue + } + if want := `"` + test.want + `"`; string(out) != want { + t.Errorf("%d: MarshalJSON output mismatch: got %q, want %q", in, out, want) + continue + } + if out := (Uint)(in).String(); out != test.want { + t.Errorf("%x: String mismatch: got %q, want %q", in, out, test.want) + continue + } + } +} + +var ( + // These are variables (not constants) to avoid constant overflow + // checks in the compiler on 32bit platforms. + maxUint33bits = uint64(^uint32(0)) + 1 + maxUint64bits = ^uint64(0) +) + +var unmarshalUintTests = []unmarshalTest{ + // invalid encoding + {input: "", wantErr: errJSONEOF}, + {input: "null", wantErr: errNonString(uintT)}, + {input: "10", wantErr: errNonString(uintT)}, + {input: `"0"`, wantErr: wrapTypeError(ErrMissingPrefix, uintT)}, + {input: `"0x"`, wantErr: wrapTypeError(ErrEmptyNumber, uintT)}, + {input: `"0x01"`, wantErr: wrapTypeError(ErrLeadingZero, uintT)}, + {input: `"0x100000000"`, want: uint(maxUint33bits), wantErr32bit: wrapTypeError(ErrUintRange, uintT)}, + {input: `"0xfffffffffffffffff"`, wantErr: wrapTypeError(ErrUintRange, uintT)}, + {input: `"0xx"`, wantErr: wrapTypeError(ErrSyntax, uintT)}, + {input: `"0x1zz01"`, wantErr: wrapTypeError(ErrSyntax, uintT)}, + + // valid encoding + {input: `""`, want: uint(0)}, + {input: `"0x0"`, want: uint(0)}, + {input: `"0x2"`, want: uint(0x2)}, + {input: `"0x2F2"`, want: uint(0x2f2)}, + {input: `"0X2F2"`, want: uint(0x2f2)}, + {input: `"0x1122aaff"`, want: uint(0x1122aaff)}, + {input: `"0xbbb"`, want: uint(0xbbb)}, + {input: `"0xffffffff"`, want: uint(0xffffffff)}, + {input: `"0xffffffffffffffff"`, want: uint(maxUint64bits), wantErr32bit: wrapTypeError(ErrUintRange, uintT)}, +} + +func TestUnmarshalUint(t *testing.T) { + for _, test := range unmarshalUintTests { + var v Uint + err := json.Unmarshal([]byte(test.input), &v) + if uintBits == 32 && test.wantErr32bit != nil { + checkError(t, test.input, err, test.wantErr32bit) + continue + } + if !checkError(t, test.input, err, test.wantErr) { + continue + } + if uint(v) != test.want.(uint) { + t.Errorf("input %s: value mismatch: got %d, want %d", test.input, v, test.want) + continue + } + } +} + +func TestUnmarshalFixedUnprefixedText(t *testing.T) { + tests := []struct { + input string + want []byte + wantErr error + }{ + {input: "0x2", wantErr: ErrOddLength}, + {input: "2", wantErr: ErrOddLength}, + {input: "4444", wantErr: errors.New("hex string has length 4, want 8 for x")}, + {input: "4444", wantErr: errors.New("hex string has length 4, want 8 for x")}, + // check that output is not modified for partially correct input + {input: "444444gg", wantErr: ErrSyntax, want: []byte{0, 0, 0, 0}}, + {input: "0x444444gg", wantErr: ErrSyntax, want: []byte{0, 0, 0, 0}}, + // valid inputs + {input: "44444444", want: []byte{0x44, 0x44, 0x44, 0x44}}, + {input: "0x44444444", want: []byte{0x44, 0x44, 0x44, 0x44}}, + } + + for _, test := range tests { + out := make([]byte, 4) + err := UnmarshalFixedUnprefixedText("x", []byte(test.input), out) + switch { + case err == nil && test.wantErr != nil: + t.Errorf("%q: got no error, expected %q", test.input, test.wantErr) + case err != nil && test.wantErr == nil: + t.Errorf("%q: unexpected error %q", test.input, err) + case err != nil && err.Error() != test.wantErr.Error(): + t.Errorf("%q: error mismatch: got %q, want %q", test.input, err, test.wantErr) + } + if test.want != nil && !bytes.Equal(out, test.want) { + t.Errorf("%q: output mismatch: got %x, want %x", test.input, out, test.want) + } + } +} diff --git a/encoding/common/math/big.go b/encoding/common/math/big.go new file mode 100644 index 000000000..1af5b4d87 --- /dev/null +++ b/encoding/common/math/big.go @@ -0,0 +1,259 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Package math provides integer math utilities. +package math + +import ( + "fmt" + "math/big" +) + +// Various big integer limit values. +var ( + tt255 = BigPow(2, 255) + tt256 = BigPow(2, 256) + tt256m1 = new(big.Int).Sub(tt256, big.NewInt(1)) + tt63 = BigPow(2, 63) + MaxBig256 = new(big.Int).Set(tt256m1) + MaxBig63 = new(big.Int).Sub(tt63, big.NewInt(1)) +) + +const ( + // number of bits in a big.Word + wordBits = 32 << (uint64(^big.Word(0)) >> 63) + // number of bytes in a big.Word + wordBytes = wordBits / 8 +) + +// HexOrDecimal256 marshals big.Int as hex or decimal. +type HexOrDecimal256 big.Int + +// NewHexOrDecimal256 creates a new HexOrDecimal256 +func NewHexOrDecimal256(x int64) *HexOrDecimal256 { + b := big.NewInt(x) + h := HexOrDecimal256(*b) + return &h +} + +// UnmarshalText implements encoding.TextUnmarshaler. +func (i *HexOrDecimal256) UnmarshalText(input []byte) error { + bigint, ok := ParseBig256(string(input)) + if !ok { + return fmt.Errorf("invalid hex or decimal integer %q", input) + } + *i = HexOrDecimal256(*bigint) + return nil +} + +// MarshalText implements encoding.TextMarshaler. +func (i *HexOrDecimal256) MarshalText() ([]byte, error) { + if i == nil { + return []byte("0x0"), nil + } + return []byte(fmt.Sprintf("%#x", (*big.Int)(i))), nil +} + +// Decimal256 unmarshals big.Int as a decimal string. When unmarshalling, +// it however accepts either "0x"-prefixed (hex encoded) or non-prefixed (decimal) +type Decimal256 big.Int + +// NewHexOrDecimal256 creates a new Decimal256 +func NewDecimal256(x int64) *Decimal256 { + b := big.NewInt(x) + d := Decimal256(*b) + return &d +} + +// UnmarshalText implements encoding.TextUnmarshaler. +func (i *Decimal256) UnmarshalText(input []byte) error { + bigint, ok := ParseBig256(string(input)) + if !ok { + return fmt.Errorf("invalid hex or decimal integer %q", input) + } + *i = Decimal256(*bigint) + return nil +} + +// MarshalText implements encoding.TextMarshaler. +func (i *Decimal256) MarshalText() ([]byte, error) { + return []byte(i.String()), nil +} + +// String implements Stringer. +func (i *Decimal256) String() string { + if i == nil { + return "0" + } + return fmt.Sprintf("%#d", (*big.Int)(i)) +} + +// ParseBig256 parses s as a 256 bit integer in decimal or hexadecimal syntax. +// Leading zeros are accepted. The empty string parses as zero. +func ParseBig256(s string) (*big.Int, bool) { + if s == "" { + return new(big.Int), true + } + var bigint *big.Int + var ok bool + if len(s) >= 2 && (s[:2] == "0x" || s[:2] == "0X") { + bigint, ok = new(big.Int).SetString(s[2:], 16) + } else { + bigint, ok = new(big.Int).SetString(s, 10) + } + if ok && bigint.BitLen() > 256 { + bigint, ok = nil, false + } + return bigint, ok +} + +// MustParseBig256 parses s as a 256 bit big integer and panics if the string is invalid. +func MustParseBig256(s string) *big.Int { + v, ok := ParseBig256(s) + if !ok { + panic("invalid 256 bit integer: " + s) + } + return v +} + +// BigPow returns a ** b as a big integer. +func BigPow(a, b int64) *big.Int { + r := big.NewInt(a) + return r.Exp(r, big.NewInt(b), nil) +} + +// BigMax returns the larger of x or y. +func BigMax(x, y *big.Int) *big.Int { + if x.Cmp(y) < 0 { + return y + } + return x +} + +// BigMin returns the smaller of x or y. +func BigMin(x, y *big.Int) *big.Int { + if x.Cmp(y) > 0 { + return y + } + return x +} + +// FirstBitSet returns the index of the first 1 bit in v, counting from LSB. +func FirstBitSet(v *big.Int) int { + for i := 0; i < v.BitLen(); i++ { + if v.Bit(i) > 0 { + return i + } + } + return v.BitLen() +} + +// PaddedBigBytes encodes a big integer as a big-endian byte slice. The length +// of the slice is at least n bytes. +func PaddedBigBytes(bigint *big.Int, n int) []byte { + if bigint.BitLen()/8 >= n { + return bigint.Bytes() + } + ret := make([]byte, n) + ReadBits(bigint, ret) + return ret +} + +// bigEndianByteAt returns the byte at position n, +// in Big-Endian encoding +// So n==0 returns the least significant byte +func bigEndianByteAt(bigint *big.Int, n int) byte { + words := bigint.Bits() + // Check word-bucket the byte will reside in + i := n / wordBytes + if i >= len(words) { + return byte(0) + } + word := words[i] + // Offset of the byte + shift := 8 * uint(n%wordBytes) + + return byte(word >> shift) +} + +// Byte returns the byte at position n, +// with the supplied padlength in Little-Endian encoding. +// n==0 returns the MSB +// Example: bigint '5', padlength 32, n=31 => 5 +func Byte(bigint *big.Int, padlength, n int) byte { + if n >= padlength { + return byte(0) + } + return bigEndianByteAt(bigint, padlength-1-n) +} + +// ReadBits encodes the absolute value of bigint as big-endian bytes. Callers must ensure +// that buf has enough space. If buf is too short the result will be incomplete. +func ReadBits(bigint *big.Int, buf []byte) { + i := len(buf) + for _, d := range bigint.Bits() { + for j := 0; j < wordBytes && i > 0; j++ { + i-- + buf[i] = byte(d) + d >>= 8 + } + } +} + +// U256 encodes as a 256 bit two's complement number. This operation is destructive. +func U256(x *big.Int) *big.Int { + return x.And(x, tt256m1) +} + +// U256Bytes converts a big Int into a 256bit EVM number. +// This operation is destructive. +func U256Bytes(n *big.Int) []byte { + return PaddedBigBytes(U256(n), 32) +} + +// S256 interprets x as a two's complement number. +// x must not exceed 256 bits (the result is undefined if it does) and is not modified. +// +// S256(0) = 0 +// S256(1) = 1 +// S256(2**255) = -2**255 +// S256(2**256-1) = -1 +func S256(x *big.Int) *big.Int { + if x.Cmp(tt255) < 0 { + return x + } + return new(big.Int).Sub(x, tt256) +} + +// Exp implements exponentiation by squaring. +// Exp returns a newly-allocated big integer and does not change +// base or exponent. The result is truncated to 256 bits. +// +// Courtesy @karalabe and @chfast +func Exp(base, exponent *big.Int) *big.Int { + result := big.NewInt(1) + + for _, word := range exponent.Bits() { + for i := 0; i < wordBits; i++ { + if word&1 == 1 { + U256(result.Mul(result, base)) + } + U256(base.Mul(base, base)) + word >>= 1 + } + } + return result +} diff --git a/encoding/common/math/big_test.go b/encoding/common/math/big_test.go new file mode 100644 index 000000000..238c57604 --- /dev/null +++ b/encoding/common/math/big_test.go @@ -0,0 +1,327 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package math + +import ( + "bytes" + "encoding/hex" + "math/big" + "testing" + + "github.com/onflow/flow-go-sdk/encoding/common" +) + +func TestHexOrDecimal256(t *testing.T) { + tests := []struct { + input string + num *big.Int + ok bool + }{ + {"", big.NewInt(0), true}, + {"0", big.NewInt(0), true}, + {"0x0", big.NewInt(0), true}, + {"12345678", big.NewInt(12345678), true}, + {"0x12345678", big.NewInt(0x12345678), true}, + {"0X12345678", big.NewInt(0x12345678), true}, + // Tests for leading zero behaviour: + {"0123456789", big.NewInt(123456789), true}, // note: not octal + {"00", big.NewInt(0), true}, + {"0x00", big.NewInt(0), true}, + {"0x012345678abc", big.NewInt(0x12345678abc), true}, + // Invalid syntax: + {"abcdef", nil, false}, + {"0xgg", nil, false}, + // Larger than 256 bits: + {"115792089237316195423570985008687907853269984665640564039457584007913129639936", nil, false}, + } + for _, test := range tests { + var num HexOrDecimal256 + err := num.UnmarshalText([]byte(test.input)) + if (err == nil) != test.ok { + t.Errorf("ParseBig(%q) -> (err == nil) == %t, want %t", test.input, err == nil, test.ok) + continue + } + if test.num != nil && (*big.Int)(&num).Cmp(test.num) != 0 { + t.Errorf("ParseBig(%q) -> %d, want %d", test.input, (*big.Int)(&num), test.num) + } + } +} + +func TestMustParseBig256(t *testing.T) { + defer func() { + if recover() == nil { + t.Error("MustParseBig should've panicked") + } + }() + MustParseBig256("ggg") +} + +func TestBigMax(t *testing.T) { + a := big.NewInt(10) + b := big.NewInt(5) + + max1 := BigMax(a, b) + if max1 != a { + t.Errorf("Expected %d got %d", a, max1) + } + + max2 := BigMax(b, a) + if max2 != a { + t.Errorf("Expected %d got %d", a, max2) + } +} + +func TestBigMin(t *testing.T) { + a := big.NewInt(10) + b := big.NewInt(5) + + min1 := BigMin(a, b) + if min1 != b { + t.Errorf("Expected %d got %d", b, min1) + } + + min2 := BigMin(b, a) + if min2 != b { + t.Errorf("Expected %d got %d", b, min2) + } +} + +func TestFirstBigSet(t *testing.T) { + tests := []struct { + num *big.Int + ix int + }{ + {big.NewInt(0), 0}, + {big.NewInt(1), 0}, + {big.NewInt(2), 1}, + {big.NewInt(0x100), 8}, + } + for _, test := range tests { + if ix := FirstBitSet(test.num); ix != test.ix { + t.Errorf("FirstBitSet(b%b) = %d, want %d", test.num, ix, test.ix) + } + } +} + +func TestPaddedBigBytes(t *testing.T) { + tests := []struct { + num *big.Int + n int + result []byte + }{ + {num: big.NewInt(0), n: 4, result: []byte{0, 0, 0, 0}}, + {num: big.NewInt(1), n: 4, result: []byte{0, 0, 0, 1}}, + {num: big.NewInt(512), n: 4, result: []byte{0, 0, 2, 0}}, + {num: BigPow(2, 32), n: 4, result: []byte{1, 0, 0, 0, 0}}, + } + for _, test := range tests { + if result := PaddedBigBytes(test.num, test.n); !bytes.Equal(result, test.result) { + t.Errorf("PaddedBigBytes(%d, %d) = %v, want %v", test.num, test.n, result, test.result) + } + } +} + +func BenchmarkPaddedBigBytesLargePadding(b *testing.B) { + bigint := MustParseBig256("123456789123456789123456789123456789") + for i := 0; i < b.N; i++ { + PaddedBigBytes(bigint, 200) + } +} + +func BenchmarkPaddedBigBytesSmallPadding(b *testing.B) { + bigint := MustParseBig256("0x18F8F8F1000111000110011100222004330052300000000000000000FEFCF3CC") + for i := 0; i < b.N; i++ { + PaddedBigBytes(bigint, 5) + } +} + +func BenchmarkPaddedBigBytesSmallOnePadding(b *testing.B) { + bigint := MustParseBig256("0x18F8F8F1000111000110011100222004330052300000000000000000FEFCF3CC") + for i := 0; i < b.N; i++ { + PaddedBigBytes(bigint, 32) + } +} + +func BenchmarkByteAtBrandNew(b *testing.B) { + bigint := MustParseBig256("0x18F8F8F1000111000110011100222004330052300000000000000000FEFCF3CC") + for i := 0; i < b.N; i++ { + bigEndianByteAt(bigint, 15) + } +} + +func BenchmarkByteAt(b *testing.B) { + bigint := MustParseBig256("0x18F8F8F1000111000110011100222004330052300000000000000000FEFCF3CC") + for i := 0; i < b.N; i++ { + bigEndianByteAt(bigint, 15) + } +} + +func BenchmarkByteAtOld(b *testing.B) { + + bigint := MustParseBig256("0x18F8F8F1000111000110011100222004330052300000000000000000FEFCF3CC") + for i := 0; i < b.N; i++ { + PaddedBigBytes(bigint, 32) + } +} + +func TestReadBits(t *testing.T) { + check := func(input string) { + want, _ := hex.DecodeString(input) + int, _ := new(big.Int).SetString(input, 16) + buf := make([]byte, len(want)) + ReadBits(int, buf) + if !bytes.Equal(buf, want) { + t.Errorf("have: %x\nwant: %x", buf, want) + } + } + check("000000000000000000000000000000000000000000000000000000FEFCF3F8F0") + check("0000000000012345000000000000000000000000000000000000FEFCF3F8F0") + check("18F8F8F1000111000110011100222004330052300000000000000000FEFCF3F8F0") +} + +func TestU256(t *testing.T) { + tests := []struct{ x, y *big.Int }{ + {x: big.NewInt(0), y: big.NewInt(0)}, + {x: big.NewInt(1), y: big.NewInt(1)}, + {x: BigPow(2, 255), y: BigPow(2, 255)}, + {x: BigPow(2, 256), y: big.NewInt(0)}, + {x: new(big.Int).Add(BigPow(2, 256), big.NewInt(1)), y: big.NewInt(1)}, + // negative values + {x: big.NewInt(-1), y: new(big.Int).Sub(BigPow(2, 256), big.NewInt(1))}, + {x: big.NewInt(-2), y: new(big.Int).Sub(BigPow(2, 256), big.NewInt(2))}, + {x: BigPow(2, -255), y: big.NewInt(1)}, + } + for _, test := range tests { + if y := U256(new(big.Int).Set(test.x)); y.Cmp(test.y) != 0 { + t.Errorf("U256(%x) = %x, want %x", test.x, y, test.y) + } + } +} + +func TestU256Bytes(t *testing.T) { + ubytes := make([]byte, 32) + ubytes[31] = 1 + + unsigned := U256Bytes(big.NewInt(1)) + if !bytes.Equal(unsigned, ubytes) { + t.Errorf("expected %x got %x", ubytes, unsigned) + } +} + +func TestBigEndianByteAt(t *testing.T) { + tests := []struct { + x string + y int + exp byte + }{ + {"00", 0, 0x00}, + {"01", 1, 0x00}, + {"00", 1, 0x00}, + {"01", 0, 0x01}, + {"0000000000000000000000000000000000000000000000000000000000102030", 0, 0x30}, + {"0000000000000000000000000000000000000000000000000000000000102030", 1, 0x20}, + {"ABCDEF0908070605040302010000000000000000000000000000000000000000", 31, 0xAB}, + {"ABCDEF0908070605040302010000000000000000000000000000000000000000", 32, 0x00}, + {"ABCDEF0908070605040302010000000000000000000000000000000000000000", 500, 0x00}, + } + for _, test := range tests { + v := new(big.Int).SetBytes(common.Hex2Bytes(test.x)) + actual := bigEndianByteAt(v, test.y) + if actual != test.exp { + t.Fatalf("Expected [%v] %v:th byte to be %v, was %v.", test.x, test.y, test.exp, actual) + } + + } +} +func TestLittleEndianByteAt(t *testing.T) { + tests := []struct { + x string + y int + exp byte + }{ + {"00", 0, 0x00}, + {"01", 1, 0x00}, + {"00", 1, 0x00}, + {"01", 0, 0x00}, + {"0000000000000000000000000000000000000000000000000000000000102030", 0, 0x00}, + {"0000000000000000000000000000000000000000000000000000000000102030", 1, 0x00}, + {"ABCDEF0908070605040302010000000000000000000000000000000000000000", 31, 0x00}, + {"ABCDEF0908070605040302010000000000000000000000000000000000000000", 32, 0x00}, + {"ABCDEF0908070605040302010000000000000000000000000000000000000000", 0, 0xAB}, + {"ABCDEF0908070605040302010000000000000000000000000000000000000000", 1, 0xCD}, + {"00CDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff", 0, 0x00}, + {"00CDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff", 1, 0xCD}, + {"0000000000000000000000000000000000000000000000000000000000102030", 31, 0x30}, + {"0000000000000000000000000000000000000000000000000000000000102030", 30, 0x20}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", 32, 0x0}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", 31, 0xFF}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", 0xFFFF, 0x0}, + } + for _, test := range tests { + v := new(big.Int).SetBytes(common.Hex2Bytes(test.x)) + actual := Byte(v, 32, test.y) + if actual != test.exp { + t.Fatalf("Expected [%v] %v:th byte to be %v, was %v.", test.x, test.y, test.exp, actual) + } + + } +} + +func TestS256(t *testing.T) { + tests := []struct{ x, y *big.Int }{ + {x: big.NewInt(0), y: big.NewInt(0)}, + {x: big.NewInt(1), y: big.NewInt(1)}, + {x: big.NewInt(2), y: big.NewInt(2)}, + { + x: new(big.Int).Sub(BigPow(2, 255), big.NewInt(1)), + y: new(big.Int).Sub(BigPow(2, 255), big.NewInt(1)), + }, + { + x: BigPow(2, 255), + y: new(big.Int).Neg(BigPow(2, 255)), + }, + { + x: new(big.Int).Sub(BigPow(2, 256), big.NewInt(1)), + y: big.NewInt(-1), + }, + { + x: new(big.Int).Sub(BigPow(2, 256), big.NewInt(2)), + y: big.NewInt(-2), + }, + } + for _, test := range tests { + if y := S256(test.x); y.Cmp(test.y) != 0 { + t.Errorf("S256(%x) = %x, want %x", test.x, y, test.y) + } + } +} + +func TestExp(t *testing.T) { + tests := []struct{ base, exponent, result *big.Int }{ + {base: big.NewInt(0), exponent: big.NewInt(0), result: big.NewInt(1)}, + {base: big.NewInt(1), exponent: big.NewInt(0), result: big.NewInt(1)}, + {base: big.NewInt(1), exponent: big.NewInt(1), result: big.NewInt(1)}, + {base: big.NewInt(1), exponent: big.NewInt(2), result: big.NewInt(1)}, + {base: big.NewInt(3), exponent: big.NewInt(144), result: MustParseBig256("507528786056415600719754159741696356908742250191663887263627442114881")}, + {base: big.NewInt(2), exponent: big.NewInt(255), result: MustParseBig256("57896044618658097711785492504343953926634992332820282019728792003956564819968")}, + } + for _, test := range tests { + if result := Exp(test.base, test.exponent); result.Cmp(test.result) != 0 { + t.Errorf("Exp(%d, %d) = %d, want %d", test.base, test.exponent, result, test.result) + } + } +} diff --git a/encoding/common/math/integer.go b/encoding/common/math/integer.go new file mode 100644 index 000000000..50d3eba1f --- /dev/null +++ b/encoding/common/math/integer.go @@ -0,0 +1,98 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package math + +import ( + "fmt" + "math/bits" + "strconv" +) + +// Integer limit values. +const ( + MaxInt8 = 1<<7 - 1 + MinInt8 = -1 << 7 + MaxInt16 = 1<<15 - 1 + MinInt16 = -1 << 15 + MaxInt32 = 1<<31 - 1 + MinInt32 = -1 << 31 + MaxInt64 = 1<<63 - 1 + MinInt64 = -1 << 63 + MaxUint8 = 1<<8 - 1 + MaxUint16 = 1<<16 - 1 + MaxUint32 = 1<<32 - 1 + MaxUint64 = 1<<64 - 1 +) + +// HexOrDecimal64 marshals uint64 as hex or decimal. +type HexOrDecimal64 uint64 + +// UnmarshalText implements encoding.TextUnmarshaler. +func (i *HexOrDecimal64) UnmarshalText(input []byte) error { + int, ok := ParseUint64(string(input)) + if !ok { + return fmt.Errorf("invalid hex or decimal integer %q", input) + } + *i = HexOrDecimal64(int) + return nil +} + +// MarshalText implements encoding.TextMarshaler. +func (i HexOrDecimal64) MarshalText() ([]byte, error) { + return []byte(fmt.Sprintf("%#x", uint64(i))), nil +} + +// ParseUint64 parses s as an integer in decimal or hexadecimal syntax. +// Leading zeros are accepted. The empty string parses as zero. +func ParseUint64(s string) (uint64, bool) { + if s == "" { + return 0, true + } + if len(s) >= 2 && (s[:2] == "0x" || s[:2] == "0X") { + v, err := strconv.ParseUint(s[2:], 16, 64) + return v, err == nil + } + v, err := strconv.ParseUint(s, 10, 64) + return v, err == nil +} + +// MustParseUint64 parses s as an integer and panics if the string is invalid. +func MustParseUint64(s string) uint64 { + v, ok := ParseUint64(s) + if !ok { + panic("invalid unsigned 64 bit integer: " + s) + } + return v +} + +// SafeSub returns x-y and checks for overflow. +func SafeSub(x, y uint64) (uint64, bool) { + diff, borrowOut := bits.Sub64(x, y, 0) + return diff, borrowOut != 0 +} + +// SafeAdd returns x+y and checks for overflow. +func SafeAdd(x, y uint64) (uint64, bool) { + sum, carryOut := bits.Add64(x, y, 0) + return sum, carryOut != 0 +} + +// SafeMul returns x*y and checks for overflow. +func SafeMul(x, y uint64) (uint64, bool) { + hi, lo := bits.Mul64(x, y) + return lo, hi != 0 +} diff --git a/encoding/common/math/integer_test.go b/encoding/common/math/integer_test.go new file mode 100644 index 000000000..b31c7c26c --- /dev/null +++ b/encoding/common/math/integer_test.go @@ -0,0 +1,116 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package math + +import ( + "testing" +) + +type operation byte + +const ( + sub operation = iota + add + mul +) + +func TestOverflow(t *testing.T) { + for i, test := range []struct { + x uint64 + y uint64 + overflow bool + op operation + }{ + // add operations + {MaxUint64, 1, true, add}, + {MaxUint64 - 1, 1, false, add}, + + // sub operations + {0, 1, true, sub}, + {0, 0, false, sub}, + + // mul operations + {0, 0, false, mul}, + {10, 10, false, mul}, + {MaxUint64, 2, true, mul}, + {MaxUint64, 1, false, mul}, + } { + var overflows bool + switch test.op { + case sub: + _, overflows = SafeSub(test.x, test.y) + case add: + _, overflows = SafeAdd(test.x, test.y) + case mul: + _, overflows = SafeMul(test.x, test.y) + } + + if test.overflow != overflows { + t.Errorf("%d failed. Expected test to be %v, got %v", i, test.overflow, overflows) + } + } +} + +func TestHexOrDecimal64(t *testing.T) { + tests := []struct { + input string + num uint64 + ok bool + }{ + {"", 0, true}, + {"0", 0, true}, + {"0x0", 0, true}, + {"12345678", 12345678, true}, + {"0x12345678", 0x12345678, true}, + {"0X12345678", 0x12345678, true}, + // Tests for leading zero behaviour: + {"0123456789", 123456789, true}, // note: not octal + {"0x00", 0, true}, + {"0x012345678abc", 0x12345678abc, true}, + // Invalid syntax: + {"abcdef", 0, false}, + {"0xgg", 0, false}, + // Doesn't fit into 64 bits: + {"18446744073709551617", 0, false}, + } + for _, test := range tests { + var num HexOrDecimal64 + err := num.UnmarshalText([]byte(test.input)) + if (err == nil) != test.ok { + t.Errorf("ParseUint64(%q) -> (err == nil) = %t, want %t", test.input, err == nil, test.ok) + continue + } + if err == nil && uint64(num) != test.num { + t.Errorf("ParseUint64(%q) -> %d, want %d", test.input, num, test.num) + } + } +} + +func TestMustParseUint64(t *testing.T) { + if v := MustParseUint64("12345"); v != 12345 { + t.Errorf(`MustParseUint64("12345") = %d, want 12345`, v) + } +} + +func TestMustParseUint64Panic(t *testing.T) { + defer func() { + if recover() == nil { + t.Error("MustParseBig should've panicked") + } + }() + MustParseUint64("ggg") +} diff --git a/encoding/common/mclock/mclock.go b/encoding/common/mclock/mclock.go new file mode 100644 index 000000000..c05738cf2 --- /dev/null +++ b/encoding/common/mclock/mclock.go @@ -0,0 +1,127 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Package mclock is a wrapper for a monotonic clock source +package mclock + +import ( + "time" + + _ "unsafe" // for go:linkname +) + +//go:noescape +//go:linkname nanotime runtime.nanotime +func nanotime() int64 + +// AbsTime represents absolute monotonic time. +type AbsTime int64 + +// Now returns the current absolute monotonic time. +func Now() AbsTime { + return AbsTime(nanotime()) +} + +// Add returns t + d as absolute time. +func (t AbsTime) Add(d time.Duration) AbsTime { + return t + AbsTime(d) +} + +// Sub returns t - t2 as a duration. +func (t AbsTime) Sub(t2 AbsTime) time.Duration { + return time.Duration(t - t2) +} + +// The Clock interface makes it possible to replace the monotonic system clock with +// a simulated clock. +type Clock interface { + Now() AbsTime + Sleep(time.Duration) + NewTimer(time.Duration) ChanTimer + After(time.Duration) <-chan AbsTime + AfterFunc(d time.Duration, f func()) Timer +} + +// Timer is a cancellable event created by AfterFunc. +type Timer interface { + // Stop cancels the timer. It returns false if the timer has already + // expired or been stopped. + Stop() bool +} + +// ChanTimer is a cancellable event created by NewTimer. +type ChanTimer interface { + Timer + + // The channel returned by C receives a value when the timer expires. + C() <-chan AbsTime + // Reset reschedules the timer with a new timeout. + // It should be invoked only on stopped or expired timers with drained channels. + Reset(time.Duration) +} + +// System implements Clock using the system clock. +type System struct{} + +// Now returns the current monotonic time. +func (c System) Now() AbsTime { + return Now() +} + +// Sleep blocks for the given duration. +func (c System) Sleep(d time.Duration) { + time.Sleep(d) +} + +// NewTimer creates a timer which can be rescheduled. +func (c System) NewTimer(d time.Duration) ChanTimer { + ch := make(chan AbsTime, 1) + t := time.AfterFunc(d, func() { + // This send is non-blocking because that's how time.Timer + // behaves. It doesn't matter in the happy case, but does + // when Reset is misused. + select { + case ch <- c.Now(): + default: + } + }) + return &systemTimer{t, ch} +} + +// After returns a channel which receives the current time after d has elapsed. +func (c System) After(d time.Duration) <-chan AbsTime { + ch := make(chan AbsTime, 1) + time.AfterFunc(d, func() { ch <- c.Now() }) + return ch +} + +// AfterFunc runs f on a new goroutine after the duration has elapsed. +func (c System) AfterFunc(d time.Duration, f func()) Timer { + return time.AfterFunc(d, f) +} + +type systemTimer struct { + *time.Timer + ch <-chan AbsTime +} + +func (st *systemTimer) Reset(d time.Duration) { + st.Timer.Reset(d) +} + +func (st *systemTimer) C() <-chan AbsTime { + return st.ch +} diff --git a/encoding/common/mclock/mclock.s b/encoding/common/mclock/mclock.s new file mode 100644 index 000000000..99a7a878f --- /dev/null +++ b/encoding/common/mclock/mclock.s @@ -0,0 +1 @@ +// This file exists in order to be able to use go:linkname. diff --git a/encoding/common/mclock/simclock.go b/encoding/common/mclock/simclock.go new file mode 100644 index 000000000..766ca0f87 --- /dev/null +++ b/encoding/common/mclock/simclock.go @@ -0,0 +1,209 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package mclock + +import ( + "container/heap" + "sync" + "time" +) + +// Simulated implements a virtual Clock for reproducible time-sensitive tests. It +// simulates a scheduler on a virtual timescale where actual processing takes zero time. +// +// The virtual clock doesn't advance on its own, call Run to advance it and execute timers. +// Since there is no way to influence the Go scheduler, testing timeout behaviour involving +// goroutines needs special care. A good way to test such timeouts is as follows: First +// perform the action that is supposed to time out. Ensure that the timer you want to test +// is created. Then run the clock until after the timeout. Finally observe the effect of +// the timeout using a channel or semaphore. +type Simulated struct { + now AbsTime + scheduled simTimerHeap + mu sync.RWMutex + cond *sync.Cond +} + +// simTimer implements ChanTimer on the virtual clock. +type simTimer struct { + at AbsTime + index int // position in s.scheduled + s *Simulated + do func() + ch <-chan AbsTime +} + +func (s *Simulated) init() { + if s.cond == nil { + s.cond = sync.NewCond(&s.mu) + } +} + +// Run moves the clock by the given duration, executing all timers before that duration. +func (s *Simulated) Run(d time.Duration) { + s.mu.Lock() + s.init() + + end := s.now + AbsTime(d) + var do []func() + for len(s.scheduled) > 0 && s.scheduled[0].at <= end { + ev := heap.Pop(&s.scheduled).(*simTimer) + do = append(do, ev.do) + } + s.now = end + s.mu.Unlock() + + for _, fn := range do { + fn() + } +} + +// ActiveTimers returns the number of timers that haven't fired. +func (s *Simulated) ActiveTimers() int { + s.mu.RLock() + defer s.mu.RUnlock() + + return len(s.scheduled) +} + +// WaitForTimers waits until the clock has at least n scheduled timers. +func (s *Simulated) WaitForTimers(n int) { + s.mu.Lock() + defer s.mu.Unlock() + s.init() + + for len(s.scheduled) < n { + s.cond.Wait() + } +} + +// Now returns the current virtual time. +func (s *Simulated) Now() AbsTime { + s.mu.RLock() + defer s.mu.RUnlock() + + return s.now +} + +// Sleep blocks until the clock has advanced by d. +func (s *Simulated) Sleep(d time.Duration) { + <-s.After(d) +} + +// NewTimer creates a timer which fires when the clock has advanced by d. +func (s *Simulated) NewTimer(d time.Duration) ChanTimer { + s.mu.Lock() + defer s.mu.Unlock() + + ch := make(chan AbsTime, 1) + var timer *simTimer + timer = s.schedule(d, func() { ch <- timer.at }) + timer.ch = ch + return timer +} + +// After returns a channel which receives the current time after the clock +// has advanced by d. +func (s *Simulated) After(d time.Duration) <-chan AbsTime { + return s.NewTimer(d).C() +} + +// AfterFunc runs fn after the clock has advanced by d. Unlike with the system +// clock, fn runs on the goroutine that calls Run. +func (s *Simulated) AfterFunc(d time.Duration, fn func()) Timer { + s.mu.Lock() + defer s.mu.Unlock() + + return s.schedule(d, fn) +} + +func (s *Simulated) schedule(d time.Duration, fn func()) *simTimer { + s.init() + + at := s.now + AbsTime(d) + ev := &simTimer{do: fn, at: at, s: s} + heap.Push(&s.scheduled, ev) + s.cond.Broadcast() + return ev +} + +func (ev *simTimer) Stop() bool { + ev.s.mu.Lock() + defer ev.s.mu.Unlock() + + if ev.index < 0 { + return false + } + heap.Remove(&ev.s.scheduled, ev.index) + ev.s.cond.Broadcast() + ev.index = -1 + return true +} + +func (ev *simTimer) Reset(d time.Duration) { + if ev.ch == nil { + panic("mclock: Reset() on timer created by AfterFunc") + } + + ev.s.mu.Lock() + defer ev.s.mu.Unlock() + ev.at = ev.s.now.Add(d) + if ev.index < 0 { + heap.Push(&ev.s.scheduled, ev) // already expired + } else { + heap.Fix(&ev.s.scheduled, ev.index) // hasn't fired yet, reschedule + } + ev.s.cond.Broadcast() +} + +func (ev *simTimer) C() <-chan AbsTime { + if ev.ch == nil { + panic("mclock: C() on timer created by AfterFunc") + } + return ev.ch +} + +type simTimerHeap []*simTimer + +func (h *simTimerHeap) Len() int { + return len(*h) +} + +func (h *simTimerHeap) Less(i, j int) bool { + return (*h)[i].at < (*h)[j].at +} + +func (h *simTimerHeap) Swap(i, j int) { + (*h)[i], (*h)[j] = (*h)[j], (*h)[i] + (*h)[i].index = i + (*h)[j].index = j +} + +func (h *simTimerHeap) Push(x interface{}) { + t := x.(*simTimer) + t.index = len(*h) + *h = append(*h, t) +} + +func (h *simTimerHeap) Pop() interface{} { + end := len(*h) - 1 + t := (*h)[end] + t.index = -1 + (*h)[end] = nil + *h = (*h)[:end] + return t +} diff --git a/encoding/common/mclock/simclock_test.go b/encoding/common/mclock/simclock_test.go new file mode 100644 index 000000000..582bc31dc --- /dev/null +++ b/encoding/common/mclock/simclock_test.go @@ -0,0 +1,162 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package mclock + +import ( + "testing" + "time" +) + +var _ Clock = System{} +var _ Clock = new(Simulated) + +func TestSimulatedAfter(t *testing.T) { + var ( + timeout = 30 * time.Minute + offset = 99 * time.Hour + adv = 11 * time.Minute + c Simulated + ) + c.Run(offset) + + end := c.Now().Add(timeout) + ch := c.After(timeout) + for c.Now() < end.Add(-adv) { + c.Run(adv) + select { + case <-ch: + t.Fatal("Timer fired early") + default: + } + } + + c.Run(adv) + select { + case stamp := <-ch: + want := AbsTime(0).Add(offset).Add(timeout) + if stamp != want { + t.Errorf("Wrong time sent on timer channel: got %v, want %v", stamp, want) + } + default: + t.Fatal("Timer didn't fire") + } +} + +func TestSimulatedAfterFunc(t *testing.T) { + var c Simulated + + called1 := false + timer1 := c.AfterFunc(100*time.Millisecond, func() { called1 = true }) + if c.ActiveTimers() != 1 { + t.Fatalf("%d active timers, want one", c.ActiveTimers()) + } + if fired := timer1.Stop(); !fired { + t.Fatal("Stop returned false even though timer didn't fire") + } + if c.ActiveTimers() != 0 { + t.Fatalf("%d active timers, want zero", c.ActiveTimers()) + } + if called1 { + t.Fatal("timer 1 called") + } + if fired := timer1.Stop(); fired { + t.Fatal("Stop returned true after timer was already stopped") + } + + called2 := false + timer2 := c.AfterFunc(100*time.Millisecond, func() { called2 = true }) + c.Run(50 * time.Millisecond) + if called2 { + t.Fatal("timer 2 called") + } + c.Run(51 * time.Millisecond) + if !called2 { + t.Fatal("timer 2 not called") + } + if fired := timer2.Stop(); fired { + t.Fatal("Stop returned true after timer has fired") + } +} + +func TestSimulatedSleep(t *testing.T) { + var ( + c Simulated + timeout = 1 * time.Hour + done = make(chan AbsTime, 1) + ) + go func() { + c.Sleep(timeout) + done <- c.Now() + }() + + c.WaitForTimers(1) + c.Run(2 * timeout) + select { + case stamp := <-done: + want := AbsTime(2 * timeout) + if stamp != want { + t.Errorf("Wrong time after sleep: got %v, want %v", stamp, want) + } + case <-time.After(5 * time.Second): + t.Fatal("Sleep didn't return in time") + } +} + +func TestSimulatedTimerReset(t *testing.T) { + var ( + c Simulated + timeout = 1 * time.Hour + ) + timer := c.NewTimer(timeout) + c.Run(2 * timeout) + select { + case ftime := <-timer.C(): + if ftime != AbsTime(timeout) { + t.Fatalf("wrong time %v sent on timer channel, want %v", ftime, AbsTime(timeout)) + } + default: + t.Fatal("timer didn't fire") + } + + timer.Reset(timeout) + c.Run(2 * timeout) + select { + case ftime := <-timer.C(): + if ftime != AbsTime(3*timeout) { + t.Fatalf("wrong time %v sent on timer channel, want %v", ftime, AbsTime(3*timeout)) + } + default: + t.Fatal("timer didn't fire again") + } +} + +func TestSimulatedTimerStop(t *testing.T) { + var ( + c Simulated + timeout = 1 * time.Hour + ) + timer := c.NewTimer(timeout) + c.Run(2 * timeout) + if timer.Stop() { + t.Errorf("Stop returned true for fired timer") + } + select { + case <-timer.C(): + default: + t.Fatal("timer didn't fire") + } +} diff --git a/encoding/common/path.go b/encoding/common/path.go new file mode 100644 index 000000000..69820cfe5 --- /dev/null +++ b/encoding/common/path.go @@ -0,0 +1,49 @@ +// Copyright 2014 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package common + +import ( + "fmt" + "os" + "path/filepath" + "runtime" +) + +// MakeName creates a node name that follows the ethereum convention +// for such names. It adds the operation system name and Go runtime version +// the name. +func MakeName(name, version string) string { + return fmt.Sprintf("%s/v%s/%s/%s", name, version, runtime.GOOS, runtime.Version()) +} + +// FileExist checks if a file exists at filePath. +func FileExist(filePath string) bool { + _, err := os.Stat(filePath) + if err != nil && os.IsNotExist(err) { + return false + } + + return true +} + +// AbsolutePath returns datadir + filename, or filename if it is absolute. +func AbsolutePath(datadir string, filename string) string { + if filepath.IsAbs(filename) { + return filename + } + return filepath.Join(datadir, filename) +} diff --git a/encoding/common/prque/lazyqueue.go b/encoding/common/prque/lazyqueue.go new file mode 100644 index 000000000..e7cd395c0 --- /dev/null +++ b/encoding/common/prque/lazyqueue.go @@ -0,0 +1,196 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package prque + +import ( + "container/heap" + "time" + + "github.com/onflow/flow-go-sdk/encoding/common/mclock" +) + +// LazyQueue is a priority queue data structure where priorities can change over +// time and are only evaluated on demand. +// Two callbacks are required: +// - priority evaluates the actual priority of an item +// - maxPriority gives an upper estimate for the priority in any moment between +// now and the given absolute time +// If the upper estimate is exceeded then Update should be called for that item. +// A global Refresh function should also be called periodically. +type LazyQueue struct { + clock mclock.Clock + // Items are stored in one of two internal queues ordered by estimated max + // priority until the next and the next-after-next refresh. Update and Refresh + // always places items in queue[1]. + queue [2]*sstack + popQueue *sstack + period time.Duration + maxUntil mclock.AbsTime + indexOffset int + setIndex SetIndexCallback + priority PriorityCallback + maxPriority MaxPriorityCallback + lastRefresh1, lastRefresh2 mclock.AbsTime +} + +type ( + PriorityCallback func(data interface{}) int64 // actual priority callback + MaxPriorityCallback func(data interface{}, until mclock.AbsTime) int64 // estimated maximum priority callback +) + +// NewLazyQueue creates a new lazy queue +func NewLazyQueue(setIndex SetIndexCallback, priority PriorityCallback, maxPriority MaxPriorityCallback, clock mclock.Clock, refreshPeriod time.Duration) *LazyQueue { + q := &LazyQueue{ + popQueue: newSstack(nil, false), + setIndex: setIndex, + priority: priority, + maxPriority: maxPriority, + clock: clock, + period: refreshPeriod, + lastRefresh1: clock.Now(), + lastRefresh2: clock.Now(), + } + q.Reset() + q.refresh(clock.Now()) + return q +} + +// Reset clears the contents of the queue +func (q *LazyQueue) Reset() { + q.queue[0] = newSstack(q.setIndex0, false) + q.queue[1] = newSstack(q.setIndex1, false) +} + +// Refresh performs queue re-evaluation if necessary +func (q *LazyQueue) Refresh() { + now := q.clock.Now() + for time.Duration(now-q.lastRefresh2) >= q.period*2 { + q.refresh(now) + q.lastRefresh2 = q.lastRefresh1 + q.lastRefresh1 = now + } +} + +// refresh re-evaluates items in the older queue and swaps the two queues +func (q *LazyQueue) refresh(now mclock.AbsTime) { + q.maxUntil = now + mclock.AbsTime(q.period) + for q.queue[0].Len() != 0 { + q.Push(heap.Pop(q.queue[0]).(*item).value) + } + q.queue[0], q.queue[1] = q.queue[1], q.queue[0] + q.indexOffset = 1 - q.indexOffset + q.maxUntil += mclock.AbsTime(q.period) +} + +// Push adds an item to the queue +func (q *LazyQueue) Push(data interface{}) { + heap.Push(q.queue[1], &item{data, q.maxPriority(data, q.maxUntil)}) +} + +// Update updates the upper priority estimate for the item with the given queue index +func (q *LazyQueue) Update(index int) { + q.Push(q.Remove(index)) +} + +// Pop removes and returns the item with the greatest actual priority +func (q *LazyQueue) Pop() (interface{}, int64) { + var ( + resData interface{} + resPri int64 + ) + q.MultiPop(func(data interface{}, priority int64) bool { + resData = data + resPri = priority + return false + }) + return resData, resPri +} + +// peekIndex returns the index of the internal queue where the item with the +// highest estimated priority is or -1 if both are empty +func (q *LazyQueue) peekIndex() int { + if q.queue[0].Len() != 0 { + if q.queue[1].Len() != 0 && q.queue[1].blocks[0][0].priority > q.queue[0].blocks[0][0].priority { + return 1 + } + return 0 + } + if q.queue[1].Len() != 0 { + return 1 + } + return -1 +} + +// MultiPop pops multiple items from the queue and is more efficient than calling +// Pop multiple times. Popped items are passed to the callback. MultiPop returns +// when the callback returns false or there are no more items to pop. +func (q *LazyQueue) MultiPop(callback func(data interface{}, priority int64) bool) { + nextIndex := q.peekIndex() + for nextIndex != -1 { + data := heap.Pop(q.queue[nextIndex]).(*item).value + heap.Push(q.popQueue, &item{data, q.priority(data)}) + nextIndex = q.peekIndex() + for q.popQueue.Len() != 0 && (nextIndex == -1 || q.queue[nextIndex].blocks[0][0].priority < q.popQueue.blocks[0][0].priority) { + i := heap.Pop(q.popQueue).(*item) + if !callback(i.value, i.priority) { + for q.popQueue.Len() != 0 { + q.Push(heap.Pop(q.popQueue).(*item).value) + } + return + } + nextIndex = q.peekIndex() // re-check because callback is allowed to push items back + } + } +} + +// PopItem pops the item from the queue only, dropping the associated priority value. +func (q *LazyQueue) PopItem() interface{} { + i, _ := q.Pop() + return i +} + +// Remove removes removes the item with the given index. +func (q *LazyQueue) Remove(index int) interface{} { + if index < 0 { + return nil + } + return heap.Remove(q.queue[index&1^q.indexOffset], index>>1).(*item).value +} + +// Empty checks whether the priority queue is empty. +func (q *LazyQueue) Empty() bool { + return q.queue[0].Len() == 0 && q.queue[1].Len() == 0 +} + +// Size returns the number of items in the priority queue. +func (q *LazyQueue) Size() int { + return q.queue[0].Len() + q.queue[1].Len() +} + +// setIndex0 translates internal queue item index to the virtual index space of LazyQueue +func (q *LazyQueue) setIndex0(data interface{}, index int) { + if index == -1 { + q.setIndex(data, -1) + } else { + q.setIndex(data, index+index) + } +} + +// setIndex1 translates internal queue item index to the virtual index space of LazyQueue +func (q *LazyQueue) setIndex1(data interface{}, index int) { + q.setIndex(data, index+index+1) +} diff --git a/encoding/common/prque/lazyqueue_test.go b/encoding/common/prque/lazyqueue_test.go new file mode 100644 index 000000000..0392e8a7e --- /dev/null +++ b/encoding/common/prque/lazyqueue_test.go @@ -0,0 +1,124 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package prque + +import ( + "math/rand" + "sync" + "testing" + "time" + + "github.com/onflow/flow-go-sdk/encoding/common/mclock" +) + +const ( + testItems = 1000 + testPriorityStep = 100 + testSteps = 1000000 + testStepPeriod = time.Millisecond + testQueueRefresh = time.Second + testAvgRate = float64(testPriorityStep) / float64(testItems) / float64(testStepPeriod) +) + +type lazyItem struct { + p, maxp int64 + last mclock.AbsTime + index int +} + +func testPriority(a interface{}) int64 { + return a.(*lazyItem).p +} + +func testMaxPriority(a interface{}, until mclock.AbsTime) int64 { + i := a.(*lazyItem) + dt := until - i.last + i.maxp = i.p + int64(float64(dt)*testAvgRate) + return i.maxp +} + +func testSetIndex(a interface{}, i int) { + a.(*lazyItem).index = i +} + +func TestLazyQueue(t *testing.T) { + rand.Seed(time.Now().UnixNano()) + clock := &mclock.Simulated{} + q := NewLazyQueue(testSetIndex, testPriority, testMaxPriority, clock, testQueueRefresh) + + var ( + items [testItems]lazyItem + maxPri int64 + ) + + for i := range items[:] { + items[i].p = rand.Int63n(testPriorityStep * 10) + if items[i].p > maxPri { + maxPri = items[i].p + } + items[i].index = -1 + q.Push(&items[i]) + } + + var ( + lock sync.Mutex + wg sync.WaitGroup + stopCh = make(chan chan struct{}) + ) + defer wg.Wait() + wg.Add(1) + go func() { + defer wg.Done() + for { + select { + case <-clock.After(testQueueRefresh): + lock.Lock() + q.Refresh() + lock.Unlock() + case <-stopCh: + return + } + } + }() + + for c := 0; c < testSteps; c++ { + i := rand.Intn(testItems) + lock.Lock() + items[i].p += rand.Int63n(testPriorityStep*2-1) + 1 + if items[i].p > maxPri { + maxPri = items[i].p + } + items[i].last = clock.Now() + if items[i].p > items[i].maxp { + q.Update(items[i].index) + } + if rand.Intn(100) == 0 { + p := q.PopItem().(*lazyItem) + if p.p != maxPri { + lock.Unlock() + close(stopCh) + t.Fatalf("incorrect item (best known priority %d, popped %d)", maxPri, p.p) + } + q.Push(p) + } + lock.Unlock() + clock.Run(testStepPeriod) + clock.WaitForTimers(1) + } + + close(stopCh) +} diff --git a/encoding/common/prque/prque.go b/encoding/common/prque/prque.go new file mode 100755 index 000000000..54c78b5fc --- /dev/null +++ b/encoding/common/prque/prque.go @@ -0,0 +1,83 @@ +// CookieJar - A contestant's algorithm toolbox +// Copyright (c) 2013 Peter Szilagyi. All rights reserved. +// +// CookieJar is dual licensed: use of this source code is governed by a BSD +// license that can be found in the LICENSE file. Alternatively, the CookieJar +// toolbox may be used in accordance with the terms and conditions contained +// in a signed written agreement between you and the author(s). + +// This is a duplicated and slightly modified version of "gopkg.in/karalabe/cookiejar.v2/collections/prque". + +// Package prque implements a priority queue data structure supporting arbitrary +// value types and int64 priorities. +// +// If you would like to use a min-priority queue, simply negate the priorities. +// +// Internally the queue is based on the standard heap package working on a +// sortable version of the block based stack. +package prque + +import ( + "container/heap" +) + +// Priority queue data structure. +type Prque struct { + cont *sstack +} + +// New creates a new priority queue. +func New(setIndex SetIndexCallback) *Prque { + return &Prque{newSstack(setIndex, false)} +} + +// NewWrapAround creates a new priority queue with wrap-around priority handling. +func NewWrapAround(setIndex SetIndexCallback) *Prque { + return &Prque{newSstack(setIndex, true)} +} + +// Pushes a value with a given priority into the queue, expanding if necessary. +func (p *Prque) Push(data interface{}, priority int64) { + heap.Push(p.cont, &item{data, priority}) +} + +// Peek returns the value with the greates priority but does not pop it off. +func (p *Prque) Peek() (interface{}, int64) { + item := p.cont.blocks[0][0] + return item.value, item.priority +} + +// Pops the value with the greates priority off the stack and returns it. +// Currently no shrinking is done. +func (p *Prque) Pop() (interface{}, int64) { + item := heap.Pop(p.cont).(*item) + return item.value, item.priority +} + +// Pops only the item from the queue, dropping the associated priority value. +func (p *Prque) PopItem() interface{} { + return heap.Pop(p.cont).(*item).value +} + +// Remove removes the element with the given index. +func (p *Prque) Remove(i int) interface{} { + if i < 0 { + return nil + } + return heap.Remove(p.cont, i) +} + +// Checks whether the priority queue is empty. +func (p *Prque) Empty() bool { + return p.cont.Len() == 0 +} + +// Returns the number of element in the priority queue. +func (p *Prque) Size() int { + return p.cont.Len() +} + +// Clears the contents of the priority queue. +func (p *Prque) Reset() { + *p = *New(p.cont.setIndex) +} diff --git a/encoding/common/prque/prque_test.go b/encoding/common/prque/prque_test.go new file mode 100644 index 000000000..1cffcebad --- /dev/null +++ b/encoding/common/prque/prque_test.go @@ -0,0 +1,130 @@ +// CookieJar - A contestant's algorithm toolbox +// Copyright (c) 2013 Peter Szilagyi. All rights reserved. +// +// CookieJar is dual licensed: use of this source code is governed by a BSD +// license that can be found in the LICENSE file. Alternatively, the CookieJar +// toolbox may be used in accordance with the terms and conditions contained +// in a signed written agreement between you and the author(s). + +package prque + +import ( + "math/rand" + "testing" +) + +func TestPrque(t *testing.T) { + // Generate a batch of random data and a specific priority order + size := 16 * blockSize + prio := rand.Perm(size) + data := make([]int, size) + for i := 0; i < size; i++ { + data[i] = rand.Int() + } + queue := New(nil) + for rep := 0; rep < 2; rep++ { + // Fill a priority queue with the above data + for i := 0; i < size; i++ { + queue.Push(data[i], int64(prio[i])) + if queue.Size() != i+1 { + t.Errorf("queue size mismatch: have %v, want %v.", queue.Size(), i+1) + } + } + // Create a map the values to the priorities for easier verification + dict := make(map[int64]int) + for i := 0; i < size; i++ { + dict[int64(prio[i])] = data[i] + } + // Pop out the elements in priority order and verify them + prevPrio := int64(size + 1) + for !queue.Empty() { + val, prio := queue.Pop() + if prio > prevPrio { + t.Errorf("invalid priority order: %v after %v.", prio, prevPrio) + } + prevPrio = prio + if val != dict[prio] { + t.Errorf("push/pop mismatch: have %v, want %v.", val, dict[prio]) + } + delete(dict, prio) + } + } +} + +func TestReset(t *testing.T) { + // Generate a batch of random data and a specific priority order + size := 16 * blockSize + prio := rand.Perm(size) + data := make([]int, size) + for i := 0; i < size; i++ { + data[i] = rand.Int() + } + queue := New(nil) + for rep := 0; rep < 2; rep++ { + // Fill a priority queue with the above data + for i := 0; i < size; i++ { + queue.Push(data[i], int64(prio[i])) + if queue.Size() != i+1 { + t.Errorf("queue size mismatch: have %v, want %v.", queue.Size(), i+1) + } + } + // Create a map the values to the priorities for easier verification + dict := make(map[int64]int) + for i := 0; i < size; i++ { + dict[int64(prio[i])] = data[i] + } + // Pop out half the elements in priority order and verify them + prevPrio := int64(size + 1) + for i := 0; i < size/2; i++ { + val, prio := queue.Pop() + if prio > prevPrio { + t.Errorf("invalid priority order: %v after %v.", prio, prevPrio) + } + prevPrio = prio + if val != dict[prio] { + t.Errorf("push/pop mismatch: have %v, want %v.", val, dict[prio]) + } + delete(dict, prio) + } + // Reset and ensure it's empty + queue.Reset() + if !queue.Empty() { + t.Errorf("priority queue not empty after reset: %v", queue) + } + } +} + +func BenchmarkPush(b *testing.B) { + // Create some initial data + data := make([]int, b.N) + prio := make([]int64, b.N) + for i := 0; i < len(data); i++ { + data[i] = rand.Int() + prio[i] = rand.Int63() + } + // Execute the benchmark + b.ResetTimer() + queue := New(nil) + for i := 0; i < len(data); i++ { + queue.Push(data[i], prio[i]) + } +} + +func BenchmarkPop(b *testing.B) { + // Create some initial data + data := make([]int, b.N) + prio := make([]int64, b.N) + for i := 0; i < len(data); i++ { + data[i] = rand.Int() + prio[i] = rand.Int63() + } + queue := New(nil) + for i := 0; i < len(data); i++ { + queue.Push(data[i], prio[i]) + } + // Execute the benchmark + b.ResetTimer() + for !queue.Empty() { + queue.Pop() + } +} diff --git a/encoding/common/prque/sstack.go b/encoding/common/prque/sstack.go new file mode 100755 index 000000000..b06a95413 --- /dev/null +++ b/encoding/common/prque/sstack.go @@ -0,0 +1,120 @@ +// CookieJar - A contestant's algorithm toolbox +// Copyright (c) 2013 Peter Szilagyi. All rights reserved. +// +// CookieJar is dual licensed: use of this source code is governed by a BSD +// license that can be found in the LICENSE file. Alternatively, the CookieJar +// toolbox may be used in accordance with the terms and conditions contained +// in a signed written agreement between you and the author(s). + +// This is a duplicated and slightly modified version of "gopkg.in/karalabe/cookiejar.v2/collections/prque". + +package prque + +// The size of a block of data +const blockSize = 4096 + +// A prioritized item in the sorted stack. +// +// Note: priorities can "wrap around" the int64 range, a comes before b if (a.priority - b.priority) > 0. +// The difference between the lowest and highest priorities in the queue at any point should be less than 2^63. +type item struct { + value interface{} + priority int64 +} + +// SetIndexCallback is called when the element is moved to a new index. +// Providing SetIndexCallback is optional, it is needed only if the application needs +// to delete elements other than the top one. +type SetIndexCallback func(data interface{}, index int) + +// Internal sortable stack data structure. Implements the Push and Pop ops for +// the stack (heap) functionality and the Len, Less and Swap methods for the +// sortability requirements of the heaps. +type sstack struct { + setIndex SetIndexCallback + size int + capacity int + offset int + wrapAround bool + + blocks [][]*item + active []*item +} + +// Creates a new, empty stack. +func newSstack(setIndex SetIndexCallback, wrapAround bool) *sstack { + result := new(sstack) + result.setIndex = setIndex + result.active = make([]*item, blockSize) + result.blocks = [][]*item{result.active} + result.capacity = blockSize + result.wrapAround = wrapAround + return result +} + +// Pushes a value onto the stack, expanding it if necessary. Required by +// heap.Interface. +func (s *sstack) Push(data interface{}) { + if s.size == s.capacity { + s.active = make([]*item, blockSize) + s.blocks = append(s.blocks, s.active) + s.capacity += blockSize + s.offset = 0 + } else if s.offset == blockSize { + s.active = s.blocks[s.size/blockSize] + s.offset = 0 + } + if s.setIndex != nil { + s.setIndex(data.(*item).value, s.size) + } + s.active[s.offset] = data.(*item) + s.offset++ + s.size++ +} + +// Pops a value off the stack and returns it. Currently no shrinking is done. +// Required by heap.Interface. +func (s *sstack) Pop() (res interface{}) { + s.size-- + s.offset-- + if s.offset < 0 { + s.offset = blockSize - 1 + s.active = s.blocks[s.size/blockSize] + } + res, s.active[s.offset] = s.active[s.offset], nil + if s.setIndex != nil { + s.setIndex(res.(*item).value, -1) + } + return +} + +// Returns the length of the stack. Required by sort.Interface. +func (s *sstack) Len() int { + return s.size +} + +// Compares the priority of two elements of the stack (higher is first). +// Required by sort.Interface. +func (s *sstack) Less(i, j int) bool { + a, b := s.blocks[i/blockSize][i%blockSize].priority, s.blocks[j/blockSize][j%blockSize].priority + if s.wrapAround { + return a-b > 0 + } + return a > b +} + +// Swaps two elements in the stack. Required by sort.Interface. +func (s *sstack) Swap(i, j int) { + ib, io, jb, jo := i/blockSize, i%blockSize, j/blockSize, j%blockSize + a, b := s.blocks[jb][jo], s.blocks[ib][io] + if s.setIndex != nil { + s.setIndex(a.value, i) + s.setIndex(b.value, j) + } + s.blocks[ib][io], s.blocks[jb][jo] = a, b +} + +// Resets the stack, effectively clearing its contents. +func (s *sstack) Reset() { + *s = *newSstack(s.setIndex, false) +} diff --git a/encoding/common/prque/sstack_test.go b/encoding/common/prque/sstack_test.go new file mode 100644 index 000000000..bc6298979 --- /dev/null +++ b/encoding/common/prque/sstack_test.go @@ -0,0 +1,100 @@ +// CookieJar - A contestant's algorithm toolbox +// Copyright (c) 2013 Peter Szilagyi. All rights reserved. +// +// CookieJar is dual licensed: use of this source code is governed by a BSD +// license that can be found in the LICENSE file. Alternatively, the CookieJar +// toolbox may be used in accordance with the terms and conditions contained +// in a signed written agreement between you and the author(s). + +package prque + +import ( + "math/rand" + "sort" + "testing" +) + +func TestSstack(t *testing.T) { + // Create some initial data + size := 16 * blockSize + data := make([]*item, size) + for i := 0; i < size; i++ { + data[i] = &item{rand.Int(), rand.Int63()} + } + stack := newSstack(nil, false) + for rep := 0; rep < 2; rep++ { + // Push all the data into the stack, pop out every second + secs := []*item{} + for i := 0; i < size; i++ { + stack.Push(data[i]) + if i%2 == 0 { + secs = append(secs, stack.Pop().(*item)) + } + } + rest := []*item{} + for stack.Len() > 0 { + rest = append(rest, stack.Pop().(*item)) + } + // Make sure the contents of the resulting slices are ok + for i := 0; i < size; i++ { + if i%2 == 0 && data[i] != secs[i/2] { + t.Errorf("push/pop mismatch: have %v, want %v.", secs[i/2], data[i]) + } + if i%2 == 1 && data[i] != rest[len(rest)-i/2-1] { + t.Errorf("push/pop mismatch: have %v, want %v.", rest[len(rest)-i/2-1], data[i]) + } + } + } +} + +func TestSstackSort(t *testing.T) { + // Create some initial data + size := 16 * blockSize + data := make([]*item, size) + for i := 0; i < size; i++ { + data[i] = &item{rand.Int(), int64(i)} + } + // Push all the data into the stack + stack := newSstack(nil, false) + for _, val := range data { + stack.Push(val) + } + // Sort and pop the stack contents (should reverse the order) + sort.Sort(stack) + for _, val := range data { + out := stack.Pop() + if out != val { + t.Errorf("push/pop mismatch after sort: have %v, want %v.", out, val) + } + } +} + +func TestSstackReset(t *testing.T) { + // Create some initial data + size := 16 * blockSize + data := make([]*item, size) + for i := 0; i < size; i++ { + data[i] = &item{rand.Int(), rand.Int63()} + } + stack := newSstack(nil, false) + for rep := 0; rep < 2; rep++ { + // Push all the data into the stack, pop out every second + secs := []*item{} + for i := 0; i < size; i++ { + stack.Push(data[i]) + if i%2 == 0 { + secs = append(secs, stack.Pop().(*item)) + } + } + // Reset and verify both pulled and stack contents + stack.Reset() + if stack.Len() != 0 { + t.Errorf("stack not empty after reset: %v", stack) + } + for i := 0; i < size; i++ { + if i%2 == 0 && data[i] != secs[i/2] { + t.Errorf("push/pop mismatch: have %v, want %v.", secs[i/2], data[i]) + } + } + } +} diff --git a/encoding/common/size.go b/encoding/common/size.go new file mode 100644 index 000000000..097b6304a --- /dev/null +++ b/encoding/common/size.go @@ -0,0 +1,56 @@ +// Copyright 2014 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package common + +import ( + "fmt" +) + +// StorageSize is a wrapper around a float value that supports user friendly +// formatting. +type StorageSize float64 + +// String implements the stringer interface. +func (s StorageSize) String() string { + if s > 1099511627776 { + return fmt.Sprintf("%.2f TiB", s/1099511627776) + } else if s > 1073741824 { + return fmt.Sprintf("%.2f GiB", s/1073741824) + } else if s > 1048576 { + return fmt.Sprintf("%.2f MiB", s/1048576) + } else if s > 1024 { + return fmt.Sprintf("%.2f KiB", s/1024) + } else { + return fmt.Sprintf("%.2f B", s) + } +} + +// TerminalString implements log.TerminalStringer, formatting a string for console +// output during logging. +func (s StorageSize) TerminalString() string { + if s > 1099511627776 { + return fmt.Sprintf("%.2fTiB", s/1099511627776) + } else if s > 1073741824 { + return fmt.Sprintf("%.2fGiB", s/1073741824) + } else if s > 1048576 { + return fmt.Sprintf("%.2fMiB", s/1048576) + } else if s > 1024 { + return fmt.Sprintf("%.2fKiB", s/1024) + } else { + return fmt.Sprintf("%.2fB", s) + } +} diff --git a/encoding/common/size_test.go b/encoding/common/size_test.go new file mode 100644 index 000000000..0938d483c --- /dev/null +++ b/encoding/common/size_test.go @@ -0,0 +1,38 @@ +// Copyright 2014 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package common + +import ( + "testing" +) + +func TestStorageSizeString(t *testing.T) { + tests := []struct { + size StorageSize + str string + }{ + {2381273, "2.27 MiB"}, + {2192, "2.14 KiB"}, + {12, "12.00 B"}, + } + + for _, test := range tests { + if test.size.String() != test.str { + t.Errorf("%f: got %q, want %q", float64(test.size), test.size.String(), test.str) + } + } +} diff --git a/encoding/common/test_utils.go b/encoding/common/test_utils.go new file mode 100644 index 000000000..7a175412f --- /dev/null +++ b/encoding/common/test_utils.go @@ -0,0 +1,53 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package common + +import ( + "encoding/json" + "fmt" + "os" +) + +// LoadJSON reads the given file and unmarshals its content. +func LoadJSON(file string, val interface{}) error { + content, err := os.ReadFile(file) + if err != nil { + return err + } + if err := json.Unmarshal(content, val); err != nil { + if syntaxerr, ok := err.(*json.SyntaxError); ok { + line := findLine(content, syntaxerr.Offset) + return fmt.Errorf("JSON syntax error at %v:%v: %v", file, line, err) + } + return fmt.Errorf("JSON unmarshal error in %v: %v", file, err) + } + return nil +} + +// findLine returns the line number for the given offset into data. +func findLine(data []byte, offset int64) (line int) { + line = 1 + for i, r := range string(data) { + if int64(i) >= offset { + return + } + if r == '\n' { + line++ + } + } + return +} diff --git a/encoding/common/types.go b/encoding/common/types.go new file mode 100644 index 000000000..0212fad6d --- /dev/null +++ b/encoding/common/types.go @@ -0,0 +1,428 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package common + +import ( + "bytes" + "database/sql/driver" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "math/big" + "math/rand" + "reflect" + "strings" + + "github.com/onflow/flow-go-sdk/encoding/common/hexutil" + "golang.org/x/crypto/sha3" +) + +// Lengths of hashes and addresses in bytes. +const ( + // HashLength is the expected length of the hash + HashLength = 32 + // AddressLength is the expected length of the address + AddressLength = 20 +) + +var ( + hashT = reflect.TypeOf(Hash{}) + addressT = reflect.TypeOf(Address{}) +) + +// Hash represents the 32 byte Keccak256 hash of arbitrary data. +type Hash [HashLength]byte + +// BytesToHash sets b to hash. +// If b is larger than len(h), b will be cropped from the left. +func BytesToHash(b []byte) Hash { + var h Hash + h.SetBytes(b) + return h +} + +// BigToHash sets byte representation of b to hash. +// If b is larger than len(h), b will be cropped from the left. +func BigToHash(b *big.Int) Hash { return BytesToHash(b.Bytes()) } + +// HexToHash sets byte representation of s to hash. +// If b is larger than len(h), b will be cropped from the left. +func HexToHash(s string) Hash { return BytesToHash(FromHex(s)) } + +// Bytes gets the byte representation of the underlying hash. +func (h Hash) Bytes() []byte { return h[:] } + +// Big converts a hash to a big integer. +func (h Hash) Big() *big.Int { return new(big.Int).SetBytes(h[:]) } + +// Hex converts a hash to a hex string. +func (h Hash) Hex() string { return hexutil.Encode(h[:]) } + +// TerminalString implements log.TerminalStringer, formatting a string for console +// output during logging. +func (h Hash) TerminalString() string { + return fmt.Sprintf("%x..%x", h[:3], h[29:]) +} + +// String implements the stringer interface and is used also by the logger when +// doing full logging into a file. +func (h Hash) String() string { + return h.Hex() +} + +// Format implements fmt.Formatter. +// Hash supports the %v, %s, %q, %x, %X and %d format verbs. +func (h Hash) Format(s fmt.State, c rune) { + hexb := make([]byte, 2+len(h)*2) + copy(hexb, "0x") + hex.Encode(hexb[2:], h[:]) + + switch c { + case 'x', 'X': + if !s.Flag('#') { + hexb = hexb[2:] + } + if c == 'X' { + hexb = bytes.ToUpper(hexb) + } + fallthrough + case 'v', 's': + s.Write(hexb) + case 'q': + q := []byte{'"'} + s.Write(q) + s.Write(hexb) + s.Write(q) + case 'd': + fmt.Fprint(s, ([len(h)]byte)(h)) + default: + fmt.Fprintf(s, "%%!%c(hash=%x)", c, h) + } +} + +// UnmarshalText parses a hash in hex syntax. +func (h *Hash) UnmarshalText(input []byte) error { + return hexutil.UnmarshalFixedText("Hash", input, h[:]) +} + +// UnmarshalJSON parses a hash in hex syntax. +func (h *Hash) UnmarshalJSON(input []byte) error { + return hexutil.UnmarshalFixedJSON(hashT, input, h[:]) +} + +// MarshalText returns the hex representation of h. +func (h Hash) MarshalText() ([]byte, error) { + return hexutil.Bytes(h[:]).MarshalText() +} + +// SetBytes sets the hash to the value of b. +// If b is larger than len(h), b will be cropped from the left. +func (h *Hash) SetBytes(b []byte) { + if len(b) > len(h) { + b = b[len(b)-HashLength:] + } + + copy(h[HashLength-len(b):], b) +} + +// Generate implements testing/quick.Generator. +func (h Hash) Generate(rand *rand.Rand, size int) reflect.Value { + m := rand.Intn(len(h)) + for i := len(h) - 1; i > m; i-- { + h[i] = byte(rand.Uint32()) + } + return reflect.ValueOf(h) +} + +// Scan implements Scanner for database/sql. +func (h *Hash) Scan(src interface{}) error { + srcB, ok := src.([]byte) + if !ok { + return fmt.Errorf("can't scan %T into Hash", src) + } + if len(srcB) != HashLength { + return fmt.Errorf("can't scan []byte of len %d into Hash, want %d", len(srcB), HashLength) + } + copy(h[:], srcB) + return nil +} + +// Value implements valuer for database/sql. +func (h Hash) Value() (driver.Value, error) { + return h[:], nil +} + +// ImplementsGraphQLType returns true if Hash implements the specified GraphQL type. +func (Hash) ImplementsGraphQLType(name string) bool { return name == "Bytes32" } + +// UnmarshalGraphQL unmarshals the provided GraphQL query data. +func (h *Hash) UnmarshalGraphQL(input interface{}) error { + var err error + switch input := input.(type) { + case string: + err = h.UnmarshalText([]byte(input)) + default: + err = fmt.Errorf("unexpected type %T for Hash", input) + } + return err +} + +// UnprefixedHash allows marshaling a Hash without 0x prefix. +type UnprefixedHash Hash + +// UnmarshalText decodes the hash from hex. The 0x prefix is optional. +func (h *UnprefixedHash) UnmarshalText(input []byte) error { + return hexutil.UnmarshalFixedUnprefixedText("UnprefixedHash", input, h[:]) +} + +// MarshalText encodes the hash as hex. +func (h UnprefixedHash) MarshalText() ([]byte, error) { + return []byte(hex.EncodeToString(h[:])), nil +} + +/////////// Address + +// Address represents the 20 byte address of an Ethereum account. +type Address [AddressLength]byte + +// BytesToAddress returns Address with value b. +// If b is larger than len(h), b will be cropped from the left. +func BytesToAddress(b []byte) Address { + var a Address + a.SetBytes(b) + return a +} + +// BigToAddress returns Address with byte values of b. +// If b is larger than len(h), b will be cropped from the left. +func BigToAddress(b *big.Int) Address { return BytesToAddress(b.Bytes()) } + +// HexToAddress returns Address with byte values of s. +// If s is larger than len(h), s will be cropped from the left. +func HexToAddress(s string) Address { return BytesToAddress(FromHex(s)) } + +// IsHexAddress verifies whether a string can represent a valid hex-encoded +// Ethereum address or not. +func IsHexAddress(s string) bool { + if has0xPrefix(s) { + s = s[2:] + } + return len(s) == 2*AddressLength && isHex(s) +} + +// Bytes gets the string representation of the underlying address. +func (a Address) Bytes() []byte { return a[:] } + +// Hash converts an address to a hash by left-padding it with zeros. +func (a Address) Hash() Hash { return BytesToHash(a[:]) } + +// Hex returns an EIP55-compliant hex string representation of the address. +func (a Address) Hex() string { + return string(a.checksumHex()) +} + +// String implements fmt.Stringer. +func (a Address) String() string { + return a.Hex() +} + +func (a *Address) checksumHex() []byte { + buf := a.hex() + + // compute checksum + sha := sha3.NewLegacyKeccak256() + sha.Write(buf[2:]) + hash := sha.Sum(nil) + for i := 2; i < len(buf); i++ { + hashByte := hash[(i-2)/2] + if i%2 == 0 { + hashByte = hashByte >> 4 + } else { + hashByte &= 0xf + } + if buf[i] > '9' && hashByte > 7 { + buf[i] -= 32 + } + } + return buf[:] +} + +func (a Address) hex() []byte { + var buf [len(a)*2 + 2]byte + copy(buf[:2], "0x") + hex.Encode(buf[2:], a[:]) + return buf[:] +} + +// Format implements fmt.Formatter. +// Address supports the %v, %s, %q, %x, %X and %d format verbs. +func (a Address) Format(s fmt.State, c rune) { + switch c { + case 'v', 's': + s.Write(a.checksumHex()) + case 'q': + q := []byte{'"'} + s.Write(q) + s.Write(a.checksumHex()) + s.Write(q) + case 'x', 'X': + // %x disables the checksum. + hex := a.hex() + if !s.Flag('#') { + hex = hex[2:] + } + if c == 'X' { + hex = bytes.ToUpper(hex) + } + s.Write(hex) + case 'd': + fmt.Fprint(s, ([len(a)]byte)(a)) + default: + fmt.Fprintf(s, "%%!%c(address=%x)", c, a) + } +} + +// SetBytes sets the address to the value of b. +// If b is larger than len(a), b will be cropped from the left. +func (a *Address) SetBytes(b []byte) { + if len(b) > len(a) { + b = b[len(b)-AddressLength:] + } + copy(a[AddressLength-len(b):], b) +} + +// MarshalText returns the hex representation of a. +func (a Address) MarshalText() ([]byte, error) { + return hexutil.Bytes(a[:]).MarshalText() +} + +// UnmarshalText parses a hash in hex syntax. +func (a *Address) UnmarshalText(input []byte) error { + return hexutil.UnmarshalFixedText("Address", input, a[:]) +} + +// UnmarshalJSON parses a hash in hex syntax. +func (a *Address) UnmarshalJSON(input []byte) error { + return hexutil.UnmarshalFixedJSON(addressT, input, a[:]) +} + +// Scan implements Scanner for database/sql. +func (a *Address) Scan(src interface{}) error { + srcB, ok := src.([]byte) + if !ok { + return fmt.Errorf("can't scan %T into Address", src) + } + if len(srcB) != AddressLength { + return fmt.Errorf("can't scan []byte of len %d into Address, want %d", len(srcB), AddressLength) + } + copy(a[:], srcB) + return nil +} + +// Value implements valuer for database/sql. +func (a Address) Value() (driver.Value, error) { + return a[:], nil +} + +// ImplementsGraphQLType returns true if Hash implements the specified GraphQL type. +func (a Address) ImplementsGraphQLType(name string) bool { return name == "Address" } + +// UnmarshalGraphQL unmarshals the provided GraphQL query data. +func (a *Address) UnmarshalGraphQL(input interface{}) error { + var err error + switch input := input.(type) { + case string: + err = a.UnmarshalText([]byte(input)) + default: + err = fmt.Errorf("unexpected type %T for Address", input) + } + return err +} + +// UnprefixedAddress allows marshaling an Address without 0x prefix. +type UnprefixedAddress Address + +// UnmarshalText decodes the address from hex. The 0x prefix is optional. +func (a *UnprefixedAddress) UnmarshalText(input []byte) error { + return hexutil.UnmarshalFixedUnprefixedText("UnprefixedAddress", input, a[:]) +} + +// MarshalText encodes the address as hex. +func (a UnprefixedAddress) MarshalText() ([]byte, error) { + return []byte(hex.EncodeToString(a[:])), nil +} + +// MixedcaseAddress retains the original string, which may or may not be +// correctly checksummed +type MixedcaseAddress struct { + addr Address + original string +} + +// NewMixedcaseAddress constructor (mainly for testing) +func NewMixedcaseAddress(addr Address) MixedcaseAddress { + return MixedcaseAddress{addr: addr, original: addr.Hex()} +} + +// NewMixedcaseAddressFromString is mainly meant for unit-testing +func NewMixedcaseAddressFromString(hexaddr string) (*MixedcaseAddress, error) { + if !IsHexAddress(hexaddr) { + return nil, errors.New("invalid address") + } + a := FromHex(hexaddr) + return &MixedcaseAddress{addr: BytesToAddress(a), original: hexaddr}, nil +} + +// UnmarshalJSON parses MixedcaseAddress +func (ma *MixedcaseAddress) UnmarshalJSON(input []byte) error { + if err := hexutil.UnmarshalFixedJSON(addressT, input, ma.addr[:]); err != nil { + return err + } + return json.Unmarshal(input, &ma.original) +} + +// MarshalJSON marshals the original value +func (ma *MixedcaseAddress) MarshalJSON() ([]byte, error) { + if strings.HasPrefix(ma.original, "0x") || strings.HasPrefix(ma.original, "0X") { + return json.Marshal(fmt.Sprintf("0x%s", ma.original[2:])) + } + return json.Marshal(fmt.Sprintf("0x%s", ma.original)) +} + +// Address returns the address +func (ma *MixedcaseAddress) Address() Address { + return ma.addr +} + +// String implements fmt.Stringer +func (ma *MixedcaseAddress) String() string { + if ma.ValidChecksum() { + return fmt.Sprintf("%s [chksum ok]", ma.original) + } + return fmt.Sprintf("%s [chksum INVALID]", ma.original) +} + +// ValidChecksum returns true if the address has valid checksum +func (ma *MixedcaseAddress) ValidChecksum() bool { + return ma.original == ma.addr.Hex() +} + +// Original returns the mixed-case input string +func (ma *MixedcaseAddress) Original() string { + return ma.original +} diff --git a/encoding/common/types_test.go b/encoding/common/types_test.go new file mode 100644 index 000000000..318e985f8 --- /dev/null +++ b/encoding/common/types_test.go @@ -0,0 +1,539 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package common + +import ( + "bytes" + "database/sql/driver" + "encoding/json" + "fmt" + "math/big" + "reflect" + "strings" + "testing" +) + +func TestBytesConversion(t *testing.T) { + bytes := []byte{5} + hash := BytesToHash(bytes) + + var exp Hash + exp[31] = 5 + + if hash != exp { + t.Errorf("expected %x got %x", exp, hash) + } +} + +func TestIsHexAddress(t *testing.T) { + tests := []struct { + str string + exp bool + }{ + {"0x5aaeb6053f3e94c9b9a09f33669435e7ef1beaed", true}, + {"5aaeb6053f3e94c9b9a09f33669435e7ef1beaed", true}, + {"0X5aaeb6053f3e94c9b9a09f33669435e7ef1beaed", true}, + {"0XAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", true}, + {"0xAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", true}, + {"0x5aaeb6053f3e94c9b9a09f33669435e7ef1beaed1", false}, + {"0x5aaeb6053f3e94c9b9a09f33669435e7ef1beae", false}, + {"5aaeb6053f3e94c9b9a09f33669435e7ef1beaed11", false}, + {"0xxaaeb6053f3e94c9b9a09f33669435e7ef1beaed", false}, + } + + for _, test := range tests { + if result := IsHexAddress(test.str); result != test.exp { + t.Errorf("IsHexAddress(%s) == %v; expected %v", + test.str, result, test.exp) + } + } +} + +func TestHashJsonValidation(t *testing.T) { + var tests = []struct { + Prefix string + Size int + Error string + }{ + {"", 62, "json: cannot unmarshal hex string without 0x prefix into Go value of type common.Hash"}, + {"0x", 66, "hex string has length 66, want 64 for common.Hash"}, + {"0x", 63, "json: cannot unmarshal hex string of odd length into Go value of type common.Hash"}, + {"0x", 0, "hex string has length 0, want 64 for common.Hash"}, + {"0x", 64, ""}, + {"0X", 64, ""}, + } + for _, test := range tests { + input := `"` + test.Prefix + strings.Repeat("0", test.Size) + `"` + var v Hash + err := json.Unmarshal([]byte(input), &v) + if err == nil { + if test.Error != "" { + t.Errorf("%s: error mismatch: have nil, want %q", input, test.Error) + } + } else { + if err.Error() != test.Error { + t.Errorf("%s: error mismatch: have %q, want %q", input, err, test.Error) + } + } + } +} + +func TestAddressUnmarshalJSON(t *testing.T) { + var tests = []struct { + Input string + ShouldErr bool + Output *big.Int + }{ + {"", true, nil}, + {`""`, true, nil}, + {`"0x"`, true, nil}, + {`"0x00"`, true, nil}, + {`"0xG000000000000000000000000000000000000000"`, true, nil}, + {`"0x0000000000000000000000000000000000000000"`, false, big.NewInt(0)}, + {`"0x0000000000000000000000000000000000000010"`, false, big.NewInt(16)}, + } + for i, test := range tests { + var v Address + err := json.Unmarshal([]byte(test.Input), &v) + if err != nil && !test.ShouldErr { + t.Errorf("test #%d: unexpected error: %v", i, err) + } + if err == nil { + if test.ShouldErr { + t.Errorf("test #%d: expected error, got none", i) + } + if got := new(big.Int).SetBytes(v.Bytes()); got.Cmp(test.Output) != 0 { + t.Errorf("test #%d: address mismatch: have %v, want %v", i, got, test.Output) + } + } + } +} + +func TestAddressHexChecksum(t *testing.T) { + var tests = []struct { + Input string + Output string + }{ + // Test cases from https://github.com/ethereum/EIPs/blob/master/EIPS/eip-55.md#specification + {"0x5aaeb6053f3e94c9b9a09f33669435e7ef1beaed", "0x5aAeb6053F3E94C9b9A09f33669435E7Ef1BeAed"}, + {"0xfb6916095ca1df60bb79ce92ce3ea74c37c5d359", "0xfB6916095ca1df60bB79Ce92cE3Ea74c37c5d359"}, + {"0xdbf03b407c01e7cd3cbea99509d93f8dddc8c6fb", "0xdbF03B407c01E7cD3CBea99509d93f8DDDC8C6FB"}, + {"0xd1220a0cf47c7b9be7a2e6ba89f429762e7b9adb", "0xD1220A0cf47c7B9Be7A2E6BA89F429762e7b9aDb"}, + // Ensure that non-standard length input values are handled correctly + {"0xa", "0x000000000000000000000000000000000000000A"}, + {"0x0a", "0x000000000000000000000000000000000000000A"}, + {"0x00a", "0x000000000000000000000000000000000000000A"}, + {"0x000000000000000000000000000000000000000a", "0x000000000000000000000000000000000000000A"}, + } + for i, test := range tests { + output := HexToAddress(test.Input).Hex() + if output != test.Output { + t.Errorf("test #%d: failed to match when it should (%s != %s)", i, output, test.Output) + } + } +} + +func BenchmarkAddressHex(b *testing.B) { + testAddr := HexToAddress("0x5aaeb6053f3e94c9b9a09f33669435e7ef1beaed") + for n := 0; n < b.N; n++ { + testAddr.Hex() + } +} + +func TestMixedcaseAccount_Address(t *testing.T) { + + // https://github.com/ethereum/EIPs/blob/master/EIPS/eip-55.md + // Note: 0X{checksum_addr} is not valid according to spec above + + var res []struct { + A MixedcaseAddress + Valid bool + } + if err := json.Unmarshal([]byte(`[ + {"A" : "0xae967917c465db8578ca9024c205720b1a3651A9", "Valid": false}, + {"A" : "0xAe967917c465db8578ca9024c205720b1a3651A9", "Valid": true}, + {"A" : "0XAe967917c465db8578ca9024c205720b1a3651A9", "Valid": false}, + {"A" : "0x1111111111111111111112222222222223333323", "Valid": true} + ]`), &res); err != nil { + t.Fatal(err) + } + + for _, r := range res { + if got := r.A.ValidChecksum(); got != r.Valid { + t.Errorf("Expected checksum %v, got checksum %v, input %v", r.Valid, got, r.A.String()) + } + } + + //These should throw exceptions: + var r2 []MixedcaseAddress + for _, r := range []string{ + `["0x11111111111111111111122222222222233333"]`, // Too short + `["0x111111111111111111111222222222222333332"]`, // Too short + `["0x11111111111111111111122222222222233333234"]`, // Too long + `["0x111111111111111111111222222222222333332344"]`, // Too long + `["1111111111111111111112222222222223333323"]`, // Missing 0x + `["x1111111111111111111112222222222223333323"]`, // Missing 0 + `["0xG111111111111111111112222222222223333323"]`, //Non-hex + } { + if err := json.Unmarshal([]byte(r), &r2); err == nil { + t.Errorf("Expected failure, input %v", r) + } + + } + +} + +func TestHash_Scan(t *testing.T) { + type args struct { + src interface{} + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "working scan", + args: args{src: []byte{ + 0xb2, 0x6f, 0x2b, 0x34, 0x2a, 0xab, 0x24, 0xbc, 0xf6, 0x3e, + 0xa2, 0x18, 0xc6, 0xa9, 0x27, 0x4d, 0x30, 0xab, 0x9a, 0x15, + 0xa2, 0x18, 0xc6, 0xa9, 0x27, 0x4d, 0x30, 0xab, 0x9a, 0x15, + 0x10, 0x00, + }}, + wantErr: false, + }, + { + name: "non working scan", + args: args{src: int64(1234567890)}, + wantErr: true, + }, + { + name: "invalid length scan", + args: args{src: []byte{ + 0xb2, 0x6f, 0x2b, 0x34, 0x2a, 0xab, 0x24, 0xbc, 0xf6, 0x3e, + 0xa2, 0x18, 0xc6, 0xa9, 0x27, 0x4d, 0x30, 0xab, 0x9a, 0x15, + 0xa2, 0x18, 0xc6, 0xa9, 0x27, 0x4d, 0x30, 0xab, 0x9a, 0x15, + }}, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + h := &Hash{} + if err := h.Scan(tt.args.src); (err != nil) != tt.wantErr { + t.Errorf("Hash.Scan() error = %v, wantErr %v", err, tt.wantErr) + } + + if !tt.wantErr { + for i := range h { + if h[i] != tt.args.src.([]byte)[i] { + t.Errorf( + "Hash.Scan() didn't scan the %d src correctly (have %X, want %X)", + i, h[i], tt.args.src.([]byte)[i], + ) + } + } + } + }) + } +} + +func TestHash_Value(t *testing.T) { + b := []byte{ + 0xb2, 0x6f, 0x2b, 0x34, 0x2a, 0xab, 0x24, 0xbc, 0xf6, 0x3e, + 0xa2, 0x18, 0xc6, 0xa9, 0x27, 0x4d, 0x30, 0xab, 0x9a, 0x15, + 0xa2, 0x18, 0xc6, 0xa9, 0x27, 0x4d, 0x30, 0xab, 0x9a, 0x15, + 0x10, 0x00, + } + var usedH Hash + usedH.SetBytes(b) + tests := []struct { + name string + h Hash + want driver.Value + wantErr bool + }{ + { + name: "Working value", + h: usedH, + want: b, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.h.Value() + if (err != nil) != tt.wantErr { + t.Errorf("Hash.Value() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("Hash.Value() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestAddress_Scan(t *testing.T) { + type args struct { + src interface{} + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "working scan", + args: args{src: []byte{ + 0xb2, 0x6f, 0x2b, 0x34, 0x2a, 0xab, 0x24, 0xbc, 0xf6, 0x3e, + 0xa2, 0x18, 0xc6, 0xa9, 0x27, 0x4d, 0x30, 0xab, 0x9a, 0x15, + }}, + wantErr: false, + }, + { + name: "non working scan", + args: args{src: int64(1234567890)}, + wantErr: true, + }, + { + name: "invalid length scan", + args: args{src: []byte{ + 0xb2, 0x6f, 0x2b, 0x34, 0x2a, 0xab, 0x24, 0xbc, 0xf6, 0x3e, + 0xa2, 0x18, 0xc6, 0xa9, 0x27, 0x4d, 0x30, 0xab, 0x9a, + }}, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + a := &Address{} + if err := a.Scan(tt.args.src); (err != nil) != tt.wantErr { + t.Errorf("Address.Scan() error = %v, wantErr %v", err, tt.wantErr) + } + + if !tt.wantErr { + for i := range a { + if a[i] != tt.args.src.([]byte)[i] { + t.Errorf( + "Address.Scan() didn't scan the %d src correctly (have %X, want %X)", + i, a[i], tt.args.src.([]byte)[i], + ) + } + } + } + }) + } +} + +func TestAddress_Value(t *testing.T) { + b := []byte{ + 0xb2, 0x6f, 0x2b, 0x34, 0x2a, 0xab, 0x24, 0xbc, 0xf6, 0x3e, + 0xa2, 0x18, 0xc6, 0xa9, 0x27, 0x4d, 0x30, 0xab, 0x9a, 0x15, + } + var usedA Address + usedA.SetBytes(b) + tests := []struct { + name string + a Address + want driver.Value + wantErr bool + }{ + { + name: "Working value", + a: usedA, + want: b, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.a.Value() + if (err != nil) != tt.wantErr { + t.Errorf("Address.Value() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("Address.Value() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestAddress_Format(t *testing.T) { + b := []byte{ + 0xb2, 0x6f, 0x2b, 0x34, 0x2a, 0xab, 0x24, 0xbc, 0xf6, 0x3e, + 0xa2, 0x18, 0xc6, 0xa9, 0x27, 0x4d, 0x30, 0xab, 0x9a, 0x15, + } + var addr Address + addr.SetBytes(b) + + tests := []struct { + name string + out string + want string + }{ + { + name: "println", + out: fmt.Sprintln(addr), + want: "0xB26f2b342AAb24BCF63ea218c6A9274D30Ab9A15\n", + }, + { + name: "print", + out: fmt.Sprint(addr), + want: "0xB26f2b342AAb24BCF63ea218c6A9274D30Ab9A15", + }, + { + name: "printf-s", + out: func() string { + buf := new(bytes.Buffer) + fmt.Fprintf(buf, "%s", addr) + return buf.String() + }(), + want: "0xB26f2b342AAb24BCF63ea218c6A9274D30Ab9A15", + }, + { + name: "printf-q", + out: fmt.Sprintf("%q", addr), + want: `"0xB26f2b342AAb24BCF63ea218c6A9274D30Ab9A15"`, + }, + { + name: "printf-x", + out: fmt.Sprintf("%x", addr), + want: "b26f2b342aab24bcf63ea218c6a9274d30ab9a15", + }, + { + name: "printf-X", + out: fmt.Sprintf("%X", addr), + want: "B26F2B342AAB24BCF63EA218C6A9274D30AB9A15", + }, + { + name: "printf-#x", + out: fmt.Sprintf("%#x", addr), + want: "0xb26f2b342aab24bcf63ea218c6a9274d30ab9a15", + }, + { + name: "printf-v", + out: fmt.Sprintf("%v", addr), + want: "0xB26f2b342AAb24BCF63ea218c6A9274D30Ab9A15", + }, + // The original default formatter for byte slice + { + name: "printf-d", + out: fmt.Sprintf("%d", addr), + want: "[178 111 43 52 42 171 36 188 246 62 162 24 198 169 39 77 48 171 154 21]", + }, + // Invalid format char. + { + name: "printf-t", + out: fmt.Sprintf("%t", addr), + want: "%!t(address=b26f2b342aab24bcf63ea218c6a9274d30ab9a15)", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.out != tt.want { + t.Errorf("%s does not render as expected:\n got %s\nwant %s", tt.name, tt.out, tt.want) + } + }) + } +} + +func TestHash_Format(t *testing.T) { + var hash Hash + hash.SetBytes([]byte{ + 0xb2, 0x6f, 0x2b, 0x34, 0x2a, 0xab, 0x24, 0xbc, 0xf6, 0x3e, + 0xa2, 0x18, 0xc6, 0xa9, 0x27, 0x4d, 0x30, 0xab, 0x9a, 0x15, + 0xa2, 0x18, 0xc6, 0xa9, 0x27, 0x4d, 0x30, 0xab, 0x9a, 0x15, + 0x10, 0x00, + }) + + tests := []struct { + name string + out string + want string + }{ + { + name: "println", + out: fmt.Sprintln(hash), + want: "0xb26f2b342aab24bcf63ea218c6a9274d30ab9a15a218c6a9274d30ab9a151000\n", + }, + { + name: "print", + out: fmt.Sprint(hash), + want: "0xb26f2b342aab24bcf63ea218c6a9274d30ab9a15a218c6a9274d30ab9a151000", + }, + { + name: "printf-s", + out: func() string { + buf := new(bytes.Buffer) + fmt.Fprintf(buf, "%s", hash) + return buf.String() + }(), + want: "0xb26f2b342aab24bcf63ea218c6a9274d30ab9a15a218c6a9274d30ab9a151000", + }, + { + name: "printf-q", + out: fmt.Sprintf("%q", hash), + want: `"0xb26f2b342aab24bcf63ea218c6a9274d30ab9a15a218c6a9274d30ab9a151000"`, + }, + { + name: "printf-x", + out: fmt.Sprintf("%x", hash), + want: "b26f2b342aab24bcf63ea218c6a9274d30ab9a15a218c6a9274d30ab9a151000", + }, + { + name: "printf-X", + out: fmt.Sprintf("%X", hash), + want: "B26F2B342AAB24BCF63EA218C6A9274D30AB9A15A218C6A9274D30AB9A151000", + }, + { + name: "printf-#x", + out: fmt.Sprintf("%#x", hash), + want: "0xb26f2b342aab24bcf63ea218c6a9274d30ab9a15a218c6a9274d30ab9a151000", + }, + { + name: "printf-#X", + out: fmt.Sprintf("%#X", hash), + want: "0XB26F2B342AAB24BCF63EA218C6A9274D30AB9A15A218C6A9274D30AB9A151000", + }, + { + name: "printf-v", + out: fmt.Sprintf("%v", hash), + want: "0xb26f2b342aab24bcf63ea218c6a9274d30ab9a15a218c6a9274d30ab9a151000", + }, + // The original default formatter for byte slice + { + name: "printf-d", + out: fmt.Sprintf("%d", hash), + want: "[178 111 43 52 42 171 36 188 246 62 162 24 198 169 39 77 48 171 154 21 162 24 198 169 39 77 48 171 154 21 16 0]", + }, + // Invalid format char. + { + name: "printf-t", + out: fmt.Sprintf("%t", hash), + want: "%!t(hash=b26f2b342aab24bcf63ea218c6a9274d30ab9a15a218c6a9274d30ab9a151000)", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.out != tt.want { + t.Errorf("%s does not render as expected:\n got %s\nwant %s", tt.name, tt.out, tt.want) + } + }) + } +} diff --git a/encoding/rlp/decode.go b/encoding/rlp/decode.go new file mode 100644 index 000000000..5cecd245c --- /dev/null +++ b/encoding/rlp/decode.go @@ -0,0 +1,1120 @@ +// Copyright 2014 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rlp + +import ( + "bufio" + "bytes" + "encoding/binary" + "errors" + "fmt" + "io" + "math/big" + "reflect" + "strings" + "sync" + + "github.com/onflow/flow-go-sdk/encoding/rlp/internal/rlpstruct" +) + +//lint:ignore ST1012 EOL is not an error. + +// EOL is returned when the end of the current list +// has been reached during streaming. +var EOL = errors.New("rlp: end of list") + +var ( + ErrExpectedString = errors.New("rlp: expected String or Byte") + ErrExpectedList = errors.New("rlp: expected List") + ErrCanonInt = errors.New("rlp: non-canonical integer format") + ErrCanonSize = errors.New("rlp: non-canonical size information") + ErrElemTooLarge = errors.New("rlp: element is larger than containing list") + ErrValueTooLarge = errors.New("rlp: value size exceeds available input length") + ErrMoreThanOneValue = errors.New("rlp: input contains more than one value") + + // internal errors + errNotInList = errors.New("rlp: call of ListEnd outside of any list") + errNotAtEOL = errors.New("rlp: call of ListEnd not positioned at EOL") + errUintOverflow = errors.New("rlp: uint overflow") + errNoPointer = errors.New("rlp: interface given to Decode must be a pointer") + errDecodeIntoNil = errors.New("rlp: pointer given to Decode must not be nil") + + streamPool = sync.Pool{ + New: func() interface{} { return new(Stream) }, + } +) + +// Decoder is implemented by types that require custom RLP decoding rules or need to decode +// into private fields. +// +// The DecodeRLP method should read one value from the given Stream. It is not forbidden to +// read less or more, but it might be confusing. +type Decoder interface { + DecodeRLP(*Stream) error +} + +// Decode parses RLP-encoded data from r and stores the result in the value pointed to by +// val. Please see package-level documentation for the decoding rules. Val must be a +// non-nil pointer. +// +// If r does not implement ByteReader, Decode will do its own buffering. +// +// Note that Decode does not set an input limit for all readers and may be vulnerable to +// panics cause by huge value sizes. If you need an input limit, use +// +// NewStream(r, limit).Decode(val) +func Decode(r io.Reader, val interface{}) error { + stream := streamPool.Get().(*Stream) + defer streamPool.Put(stream) + + stream.Reset(r, 0) + return stream.Decode(val) +} + +// DecodeBytes parses RLP data from b into val. Please see package-level documentation for +// the decoding rules. The input must contain exactly one value and no trailing data. +func DecodeBytes(b []byte, val interface{}) error { + r := bytes.NewReader(b) + + stream := streamPool.Get().(*Stream) + defer streamPool.Put(stream) + + stream.Reset(r, uint64(len(b))) + if err := stream.Decode(val); err != nil { + return err + } + if r.Len() > 0 { + return ErrMoreThanOneValue + } + return nil +} + +type decodeError struct { + msg string + typ reflect.Type + ctx []string +} + +func (err *decodeError) Error() string { + ctx := "" + if len(err.ctx) > 0 { + ctx = ", decoding into " + for i := len(err.ctx) - 1; i >= 0; i-- { + ctx += err.ctx[i] + } + } + return fmt.Sprintf("rlp: %s for %v%s", err.msg, err.typ, ctx) +} + +func wrapStreamError(err error, typ reflect.Type) error { + switch err { + case ErrCanonInt: + return &decodeError{msg: "non-canonical integer (leading zero bytes)", typ: typ} + case ErrCanonSize: + return &decodeError{msg: "non-canonical size information", typ: typ} + case ErrExpectedList: + return &decodeError{msg: "expected input list", typ: typ} + case ErrExpectedString: + return &decodeError{msg: "expected input string or byte", typ: typ} + case errUintOverflow: + return &decodeError{msg: "input string too long", typ: typ} + case errNotAtEOL: + return &decodeError{msg: "input list has too many elements", typ: typ} + } + return err +} + +func addErrorContext(err error, ctx string) error { + if decErr, ok := err.(*decodeError); ok { + decErr.ctx = append(decErr.ctx, ctx) + } + return err +} + +var ( + decoderInterface = reflect.TypeOf(new(Decoder)).Elem() + bigInt = reflect.TypeOf(big.Int{}) +) + +func makeDecoder(typ reflect.Type, tags rlpstruct.Tags) (dec decoder, err error) { + kind := typ.Kind() + switch { + case typ == rawValueType: + return decodeRawValue, nil + case typ.AssignableTo(reflect.PtrTo(bigInt)): + return decodeBigInt, nil + case typ.AssignableTo(bigInt): + return decodeBigIntNoPtr, nil + case kind == reflect.Ptr: + return makePtrDecoder(typ, tags) + case reflect.PtrTo(typ).Implements(decoderInterface): + return decodeDecoder, nil + case isUint(kind): + return decodeUint, nil + case kind == reflect.Bool: + return decodeBool, nil + case kind == reflect.String: + return decodeString, nil + case kind == reflect.Slice || kind == reflect.Array: + return makeListDecoder(typ, tags) + case kind == reflect.Struct: + return makeStructDecoder(typ) + case kind == reflect.Interface: + return decodeInterface, nil + default: + return nil, fmt.Errorf("rlp: type %v is not RLP-serializable", typ) + } +} + +func decodeRawValue(s *Stream, val reflect.Value) error { + r, err := s.Raw() + if err != nil { + return err + } + val.SetBytes(r) + return nil +} + +func decodeUint(s *Stream, val reflect.Value) error { + typ := val.Type() + num, err := s.uint(typ.Bits()) + if err != nil { + return wrapStreamError(err, val.Type()) + } + val.SetUint(num) + return nil +} + +func decodeBool(s *Stream, val reflect.Value) error { + b, err := s.Bool() + if err != nil { + return wrapStreamError(err, val.Type()) + } + val.SetBool(b) + return nil +} + +func decodeString(s *Stream, val reflect.Value) error { + b, err := s.Bytes() + if err != nil { + return wrapStreamError(err, val.Type()) + } + val.SetString(string(b)) + return nil +} + +func decodeBigIntNoPtr(s *Stream, val reflect.Value) error { + return decodeBigInt(s, val.Addr()) +} + +func decodeBigInt(s *Stream, val reflect.Value) error { + i := val.Interface().(*big.Int) + if i == nil { + i = new(big.Int) + val.Set(reflect.ValueOf(i)) + } + + err := s.decodeBigInt(i) + if err != nil { + return wrapStreamError(err, val.Type()) + } + return nil +} + +func makeListDecoder(typ reflect.Type, tag rlpstruct.Tags) (decoder, error) { + etype := typ.Elem() + if etype.Kind() == reflect.Uint8 && !reflect.PtrTo(etype).Implements(decoderInterface) { + if typ.Kind() == reflect.Array { + return decodeByteArray, nil + } + return decodeByteSlice, nil + } + etypeinfo := theTC.infoWhileGenerating(etype, rlpstruct.Tags{}) + if etypeinfo.decoderErr != nil { + return nil, etypeinfo.decoderErr + } + var dec decoder + switch { + case typ.Kind() == reflect.Array: + dec = func(s *Stream, val reflect.Value) error { + return decodeListArray(s, val, etypeinfo.decoder) + } + case tag.Tail: + // A slice with "tail" tag can occur as the last field + // of a struct and is supposed to swallow all remaining + // list elements. The struct decoder already called s.List, + // proceed directly to decoding the elements. + dec = func(s *Stream, val reflect.Value) error { + return decodeSliceElems(s, val, etypeinfo.decoder) + } + default: + dec = func(s *Stream, val reflect.Value) error { + return decodeListSlice(s, val, etypeinfo.decoder) + } + } + return dec, nil +} + +func decodeListSlice(s *Stream, val reflect.Value, elemdec decoder) error { + size, err := s.List() + if err != nil { + return wrapStreamError(err, val.Type()) + } + if size == 0 { + val.Set(reflect.MakeSlice(val.Type(), 0, 0)) + return s.ListEnd() + } + if err := decodeSliceElems(s, val, elemdec); err != nil { + return err + } + return s.ListEnd() +} + +func decodeSliceElems(s *Stream, val reflect.Value, elemdec decoder) error { + i := 0 + for ; ; i++ { + // grow slice if necessary + if i >= val.Cap() { + newcap := val.Cap() + val.Cap()/2 + if newcap < 4 { + newcap = 4 + } + newv := reflect.MakeSlice(val.Type(), val.Len(), newcap) + reflect.Copy(newv, val) + val.Set(newv) + } + if i >= val.Len() { + val.SetLen(i + 1) + } + // decode into element + if err := elemdec(s, val.Index(i)); err == EOL { + break + } else if err != nil { + return addErrorContext(err, fmt.Sprint("[", i, "]")) + } + } + if i < val.Len() { + val.SetLen(i) + } + return nil +} + +func decodeListArray(s *Stream, val reflect.Value, elemdec decoder) error { + if _, err := s.List(); err != nil { + return wrapStreamError(err, val.Type()) + } + vlen := val.Len() + i := 0 + for ; i < vlen; i++ { + if err := elemdec(s, val.Index(i)); err == EOL { + break + } else if err != nil { + return addErrorContext(err, fmt.Sprint("[", i, "]")) + } + } + if i < vlen { + return &decodeError{msg: "input list has too few elements", typ: val.Type()} + } + return wrapStreamError(s.ListEnd(), val.Type()) +} + +func decodeByteSlice(s *Stream, val reflect.Value) error { + b, err := s.Bytes() + if err != nil { + return wrapStreamError(err, val.Type()) + } + val.SetBytes(b) + return nil +} + +func decodeByteArray(s *Stream, val reflect.Value) error { + kind, size, err := s.Kind() + if err != nil { + return err + } + slice := byteArrayBytes(val, val.Len()) + switch kind { + case Byte: + if len(slice) == 0 { + return &decodeError{msg: "input string too long", typ: val.Type()} + } else if len(slice) > 1 { + return &decodeError{msg: "input string too short", typ: val.Type()} + } + slice[0] = s.byteval + s.kind = -1 + case String: + if uint64(len(slice)) < size { + return &decodeError{msg: "input string too long", typ: val.Type()} + } + if uint64(len(slice)) > size { + return &decodeError{msg: "input string too short", typ: val.Type()} + } + if err := s.readFull(slice); err != nil { + return err + } + // Reject cases where single byte encoding should have been used. + if size == 1 && slice[0] < 128 { + return wrapStreamError(ErrCanonSize, val.Type()) + } + case List: + return wrapStreamError(ErrExpectedString, val.Type()) + } + return nil +} + +func makeStructDecoder(typ reflect.Type) (decoder, error) { + fields, err := structFields(typ) + if err != nil { + return nil, err + } + for _, f := range fields { + if f.info.decoderErr != nil { + return nil, structFieldError{typ, f.index, f.info.decoderErr} + } + } + dec := func(s *Stream, val reflect.Value) (err error) { + if _, err := s.List(); err != nil { + return wrapStreamError(err, typ) + } + for i, f := range fields { + err := f.info.decoder(s, val.Field(f.index)) + if err == EOL { + if f.optional { + // The field is optional, so reaching the end of the list before + // reaching the last field is acceptable. All remaining undecoded + // fields are zeroed. + zeroFields(val, fields[i:]) + break + } + return &decodeError{msg: "too few elements", typ: typ} + } else if err != nil { + return addErrorContext(err, "."+typ.Field(f.index).Name) + } + } + return wrapStreamError(s.ListEnd(), typ) + } + return dec, nil +} + +func zeroFields(structval reflect.Value, fields []field) { + for _, f := range fields { + fv := structval.Field(f.index) + fv.Set(reflect.Zero(fv.Type())) + } +} + +// makePtrDecoder creates a decoder that decodes into the pointer's element type. +func makePtrDecoder(typ reflect.Type, tag rlpstruct.Tags) (decoder, error) { + etype := typ.Elem() + etypeinfo := theTC.infoWhileGenerating(etype, rlpstruct.Tags{}) + switch { + case etypeinfo.decoderErr != nil: + return nil, etypeinfo.decoderErr + case !tag.NilOK: + return makeSimplePtrDecoder(etype, etypeinfo), nil + default: + return makeNilPtrDecoder(etype, etypeinfo, tag), nil + } +} + +func makeSimplePtrDecoder(etype reflect.Type, etypeinfo *typeinfo) decoder { + return func(s *Stream, val reflect.Value) (err error) { + newval := val + if val.IsNil() { + newval = reflect.New(etype) + } + if err = etypeinfo.decoder(s, newval.Elem()); err == nil { + val.Set(newval) + } + return err + } +} + +// makeNilPtrDecoder creates a decoder that decodes empty values as nil. Non-empty +// values are decoded into a value of the element type, just like makePtrDecoder does. +// +// This decoder is used for pointer-typed struct fields with struct tag "nil". +func makeNilPtrDecoder(etype reflect.Type, etypeinfo *typeinfo, ts rlpstruct.Tags) decoder { + typ := reflect.PtrTo(etype) + nilPtr := reflect.Zero(typ) + + // Determine the value kind that results in nil pointer. + nilKind := typeNilKind(etype, ts) + + return func(s *Stream, val reflect.Value) (err error) { + kind, size, err := s.Kind() + if err != nil { + val.Set(nilPtr) + return wrapStreamError(err, typ) + } + // Handle empty values as a nil pointer. + if kind != Byte && size == 0 { + if kind != nilKind { + return &decodeError{ + msg: fmt.Sprintf("wrong kind of empty value (got %v, want %v)", kind, nilKind), + typ: typ, + } + } + // rearm s.Kind. This is important because the input + // position must advance to the next value even though + // we don't read anything. + s.kind = -1 + val.Set(nilPtr) + return nil + } + newval := val + if val.IsNil() { + newval = reflect.New(etype) + } + if err = etypeinfo.decoder(s, newval.Elem()); err == nil { + val.Set(newval) + } + return err + } +} + +var ifsliceType = reflect.TypeOf([]interface{}{}) + +func decodeInterface(s *Stream, val reflect.Value) error { + if val.Type().NumMethod() != 0 { + return fmt.Errorf("rlp: type %v is not RLP-serializable", val.Type()) + } + kind, _, err := s.Kind() + if err != nil { + return err + } + if kind == List { + slice := reflect.New(ifsliceType).Elem() + if err := decodeListSlice(s, slice, decodeInterface); err != nil { + return err + } + val.Set(slice) + } else { + b, err := s.Bytes() + if err != nil { + return err + } + val.Set(reflect.ValueOf(b)) + } + return nil +} + +func decodeDecoder(s *Stream, val reflect.Value) error { + return val.Addr().Interface().(Decoder).DecodeRLP(s) +} + +// Kind represents the kind of value contained in an RLP stream. +type Kind int8 + +const ( + Byte Kind = iota + String + List +) + +func (k Kind) String() string { + switch k { + case Byte: + return "Byte" + case String: + return "String" + case List: + return "List" + default: + return fmt.Sprintf("Unknown(%d)", k) + } +} + +// ByteReader must be implemented by any input reader for a Stream. It +// is implemented by e.g. bufio.Reader and bytes.Reader. +type ByteReader interface { + io.Reader + io.ByteReader +} + +// Stream can be used for piecemeal decoding of an input stream. This +// is useful if the input is very large or if the decoding rules for a +// type depend on the input structure. Stream does not keep an +// internal buffer. After decoding a value, the input reader will be +// positioned just before the type information for the next value. +// +// When decoding a list and the input position reaches the declared +// length of the list, all operations will return error EOL. +// The end of the list must be acknowledged using ListEnd to continue +// reading the enclosing list. +// +// Stream is not safe for concurrent use. +type Stream struct { + r ByteReader + + remaining uint64 // number of bytes remaining to be read from r + size uint64 // size of value ahead + kinderr error // error from last readKind + stack []uint64 // list sizes + uintbuf [32]byte // auxiliary buffer for integer decoding + kind Kind // kind of value ahead + byteval byte // value of single byte in type tag + limited bool // true if input limit is in effect +} + +// NewStream creates a new decoding stream reading from r. +// +// If r implements the ByteReader interface, Stream will +// not introduce any buffering. +// +// For non-toplevel values, Stream returns ErrElemTooLarge +// for values that do not fit into the enclosing list. +// +// Stream supports an optional input limit. If a limit is set, the +// size of any toplevel value will be checked against the remaining +// input length. Stream operations that encounter a value exceeding +// the remaining input length will return ErrValueTooLarge. The limit +// can be set by passing a non-zero value for inputLimit. +// +// If r is a bytes.Reader or strings.Reader, the input limit is set to +// the length of r's underlying data unless an explicit limit is +// provided. +func NewStream(r io.Reader, inputLimit uint64) *Stream { + s := new(Stream) + s.Reset(r, inputLimit) + return s +} + +// NewListStream creates a new stream that pretends to be positioned +// at an encoded list of the given length. +func NewListStream(r io.Reader, len uint64) *Stream { + s := new(Stream) + s.Reset(r, len) + s.kind = List + s.size = len + return s +} + +// Bytes reads an RLP string and returns its contents as a byte slice. +// If the input does not contain an RLP string, the returned +// error will be ErrExpectedString. +func (s *Stream) Bytes() ([]byte, error) { + kind, size, err := s.Kind() + if err != nil { + return nil, err + } + switch kind { + case Byte: + s.kind = -1 // rearm Kind + return []byte{s.byteval}, nil + case String: + b := make([]byte, size) + if err = s.readFull(b); err != nil { + return nil, err + } + if size == 1 && b[0] < 128 { + return nil, ErrCanonSize + } + return b, nil + default: + return nil, ErrExpectedString + } +} + +// ReadBytes decodes the next RLP value and stores the result in b. +// The value size must match len(b) exactly. +func (s *Stream) ReadBytes(b []byte) error { + kind, size, err := s.Kind() + if err != nil { + return err + } + switch kind { + case Byte: + if len(b) != 1 { + return fmt.Errorf("input value has wrong size 1, want %d", len(b)) + } + b[0] = s.byteval + s.kind = -1 // rearm Kind + return nil + case String: + if uint64(len(b)) != size { + return fmt.Errorf("input value has wrong size %d, want %d", size, len(b)) + } + if err = s.readFull(b); err != nil { + return err + } + if size == 1 && b[0] < 128 { + return ErrCanonSize + } + return nil + default: + return ErrExpectedString + } +} + +// Raw reads a raw encoded value including RLP type information. +func (s *Stream) Raw() ([]byte, error) { + kind, size, err := s.Kind() + if err != nil { + return nil, err + } + if kind == Byte { + s.kind = -1 // rearm Kind + return []byte{s.byteval}, nil + } + // The original header has already been read and is no longer + // available. Read content and put a new header in front of it. + start := headsize(size) + buf := make([]byte, uint64(start)+size) + if err := s.readFull(buf[start:]); err != nil { + return nil, err + } + if kind == String { + puthead(buf, 0x80, 0xB7, size) + } else { + puthead(buf, 0xC0, 0xF7, size) + } + return buf, nil +} + +// Uint reads an RLP string of up to 8 bytes and returns its contents +// as an unsigned integer. If the input does not contain an RLP string, the +// returned error will be ErrExpectedString. +// +// Deprecated: use s.Uint64 instead. +func (s *Stream) Uint() (uint64, error) { + return s.uint(64) +} + +func (s *Stream) Uint64() (uint64, error) { + return s.uint(64) +} + +func (s *Stream) Uint32() (uint32, error) { + i, err := s.uint(32) + return uint32(i), err +} + +func (s *Stream) Uint16() (uint16, error) { + i, err := s.uint(16) + return uint16(i), err +} + +func (s *Stream) Uint8() (uint8, error) { + i, err := s.uint(8) + return uint8(i), err +} + +func (s *Stream) uint(maxbits int) (uint64, error) { + kind, size, err := s.Kind() + if err != nil { + return 0, err + } + switch kind { + case Byte: + if s.byteval == 0 { + return 0, ErrCanonInt + } + s.kind = -1 // rearm Kind + return uint64(s.byteval), nil + case String: + if size > uint64(maxbits/8) { + return 0, errUintOverflow + } + v, err := s.readUint(byte(size)) + switch { + case err == ErrCanonSize: + // Adjust error because we're not reading a size right now. + return 0, ErrCanonInt + case err != nil: + return 0, err + case size > 0 && v < 128: + return 0, ErrCanonSize + default: + return v, nil + } + default: + return 0, ErrExpectedString + } +} + +// Bool reads an RLP string of up to 1 byte and returns its contents +// as a boolean. If the input does not contain an RLP string, the +// returned error will be ErrExpectedString. +func (s *Stream) Bool() (bool, error) { + num, err := s.uint(8) + if err != nil { + return false, err + } + switch num { + case 0: + return false, nil + case 1: + return true, nil + default: + return false, fmt.Errorf("rlp: invalid boolean value: %d", num) + } +} + +// List starts decoding an RLP list. If the input does not contain a +// list, the returned error will be ErrExpectedList. When the list's +// end has been reached, any Stream operation will return EOL. +func (s *Stream) List() (size uint64, err error) { + kind, size, err := s.Kind() + if err != nil { + return 0, err + } + if kind != List { + return 0, ErrExpectedList + } + + // Remove size of inner list from outer list before pushing the new size + // onto the stack. This ensures that the remaining outer list size will + // be correct after the matching call to ListEnd. + if inList, limit := s.listLimit(); inList { + s.stack[len(s.stack)-1] = limit - size + } + s.stack = append(s.stack, size) + s.kind = -1 + s.size = 0 + return size, nil +} + +// ListEnd returns to the enclosing list. +// The input reader must be positioned at the end of a list. +func (s *Stream) ListEnd() error { + // Ensure that no more data is remaining in the current list. + if inList, listLimit := s.listLimit(); !inList { + return errNotInList + } else if listLimit > 0 { + return errNotAtEOL + } + s.stack = s.stack[:len(s.stack)-1] // pop + s.kind = -1 + s.size = 0 + return nil +} + +// MoreDataInList reports whether the current list context contains +// more data to be read. +func (s *Stream) MoreDataInList() bool { + _, listLimit := s.listLimit() + return listLimit > 0 +} + +// BigInt decodes an arbitrary-size integer value. +func (s *Stream) BigInt() (*big.Int, error) { + i := new(big.Int) + if err := s.decodeBigInt(i); err != nil { + return nil, err + } + return i, nil +} + +func (s *Stream) decodeBigInt(dst *big.Int) error { + var buffer []byte + kind, size, err := s.Kind() + switch { + case err != nil: + return err + case kind == List: + return ErrExpectedString + case kind == Byte: + buffer = s.uintbuf[:1] + buffer[0] = s.byteval + s.kind = -1 // re-arm Kind + case size == 0: + // Avoid zero-length read. + s.kind = -1 + case size <= uint64(len(s.uintbuf)): + // For integers smaller than s.uintbuf, allocating a buffer + // can be avoided. + buffer = s.uintbuf[:size] + if err := s.readFull(buffer); err != nil { + return err + } + // Reject inputs where single byte encoding should have been used. + if size == 1 && buffer[0] < 128 { + return ErrCanonSize + } + default: + // For large integers, a temporary buffer is needed. + buffer = make([]byte, size) + if err := s.readFull(buffer); err != nil { + return err + } + } + + // Reject leading zero bytes. + if len(buffer) > 0 && buffer[0] == 0 { + return ErrCanonInt + } + // Set the integer bytes. + dst.SetBytes(buffer) + return nil +} + +// Decode decodes a value and stores the result in the value pointed +// to by val. Please see the documentation for the Decode function +// to learn about the decoding rules. +func (s *Stream) Decode(val interface{}) error { + if val == nil { + return errDecodeIntoNil + } + rval := reflect.ValueOf(val) + rtyp := rval.Type() + if rtyp.Kind() != reflect.Ptr { + return errNoPointer + } + if rval.IsNil() { + return errDecodeIntoNil + } + decoder, err := cachedDecoder(rtyp.Elem()) + if err != nil { + return err + } + + err = decoder(s, rval.Elem()) + if decErr, ok := err.(*decodeError); ok && len(decErr.ctx) > 0 { + // Add decode target type to error so context has more meaning. + decErr.ctx = append(decErr.ctx, fmt.Sprint("(", rtyp.Elem(), ")")) + } + return err +} + +// Reset discards any information about the current decoding context +// and starts reading from r. This method is meant to facilitate reuse +// of a preallocated Stream across many decoding operations. +// +// If r does not also implement ByteReader, Stream will do its own +// buffering. +func (s *Stream) Reset(r io.Reader, inputLimit uint64) { + if inputLimit > 0 { + s.remaining = inputLimit + s.limited = true + } else { + // Attempt to automatically discover + // the limit when reading from a byte slice. + switch br := r.(type) { + case *bytes.Reader: + s.remaining = uint64(br.Len()) + s.limited = true + case *bytes.Buffer: + s.remaining = uint64(br.Len()) + s.limited = true + case *strings.Reader: + s.remaining = uint64(br.Len()) + s.limited = true + default: + s.limited = false + } + } + // Wrap r with a buffer if it doesn't have one. + bufr, ok := r.(ByteReader) + if !ok { + bufr = bufio.NewReader(r) + } + s.r = bufr + // Reset the decoding context. + s.stack = s.stack[:0] + s.size = 0 + s.kind = -1 + s.kinderr = nil + s.byteval = 0 + s.uintbuf = [32]byte{} +} + +// Kind returns the kind and size of the next value in the +// input stream. +// +// The returned size is the number of bytes that make up the value. +// For kind == Byte, the size is zero because the value is +// contained in the type tag. +// +// The first call to Kind will read size information from the input +// reader and leave it positioned at the start of the actual bytes of +// the value. Subsequent calls to Kind (until the value is decoded) +// will not advance the input reader and return cached information. +func (s *Stream) Kind() (kind Kind, size uint64, err error) { + if s.kind >= 0 { + return s.kind, s.size, s.kinderr + } + + // Check for end of list. This needs to be done here because readKind + // checks against the list size, and would return the wrong error. + inList, listLimit := s.listLimit() + if inList && listLimit == 0 { + return 0, 0, EOL + } + // Read the actual size tag. + s.kind, s.size, s.kinderr = s.readKind() + if s.kinderr == nil { + // Check the data size of the value ahead against input limits. This + // is done here because many decoders require allocating an input + // buffer matching the value size. Checking it here protects those + // decoders from inputs declaring very large value size. + if inList && s.size > listLimit { + s.kinderr = ErrElemTooLarge + } else if s.limited && s.size > s.remaining { + s.kinderr = ErrValueTooLarge + } + } + return s.kind, s.size, s.kinderr +} + +func (s *Stream) readKind() (kind Kind, size uint64, err error) { + b, err := s.readByte() + if err != nil { + if len(s.stack) == 0 { + // At toplevel, Adjust the error to actual EOF. io.EOF is + // used by callers to determine when to stop decoding. + switch err { + case io.ErrUnexpectedEOF: + err = io.EOF + case ErrValueTooLarge: + err = io.EOF + } + } + return 0, 0, err + } + s.byteval = 0 + switch { + case b < 0x80: + // For a single byte whose value is in the [0x00, 0x7F] range, that byte + // is its own RLP encoding. + s.byteval = b + return Byte, 0, nil + case b < 0xB8: + // Otherwise, if a string is 0-55 bytes long, the RLP encoding consists + // of a single byte with value 0x80 plus the length of the string + // followed by the string. The range of the first byte is thus [0x80, 0xB7]. + return String, uint64(b - 0x80), nil + case b < 0xC0: + // If a string is more than 55 bytes long, the RLP encoding consists of a + // single byte with value 0xB7 plus the length of the length of the + // string in binary form, followed by the length of the string, followed + // by the string. For example, a length-1024 string would be encoded as + // 0xB90400 followed by the string. The range of the first byte is thus + // [0xB8, 0xBF]. + size, err = s.readUint(b - 0xB7) + if err == nil && size < 56 { + err = ErrCanonSize + } + return String, size, err + case b < 0xF8: + // If the total payload of a list (i.e. the combined length of all its + // items) is 0-55 bytes long, the RLP encoding consists of a single byte + // with value 0xC0 plus the length of the list followed by the + // concatenation of the RLP encodings of the items. The range of the + // first byte is thus [0xC0, 0xF7]. + return List, uint64(b - 0xC0), nil + default: + // If the total payload of a list is more than 55 bytes long, the RLP + // encoding consists of a single byte with value 0xF7 plus the length of + // the length of the payload in binary form, followed by the length of + // the payload, followed by the concatenation of the RLP encodings of + // the items. The range of the first byte is thus [0xF8, 0xFF]. + size, err = s.readUint(b - 0xF7) + if err == nil && size < 56 { + err = ErrCanonSize + } + return List, size, err + } +} + +func (s *Stream) readUint(size byte) (uint64, error) { + switch size { + case 0: + s.kind = -1 // rearm Kind + return 0, nil + case 1: + b, err := s.readByte() + return uint64(b), err + default: + buffer := s.uintbuf[:8] + for i := range buffer { + buffer[i] = 0 + } + start := int(8 - size) + if err := s.readFull(buffer[start:]); err != nil { + return 0, err + } + if buffer[start] == 0 { + // Note: readUint is also used to decode integer values. + // The error needs to be adjusted to become ErrCanonInt in this case. + return 0, ErrCanonSize + } + return binary.BigEndian.Uint64(buffer[:]), nil + } +} + +// readFull reads into buf from the underlying stream. +func (s *Stream) readFull(buf []byte) (err error) { + if err := s.willRead(uint64(len(buf))); err != nil { + return err + } + var nn, n int + for n < len(buf) && err == nil { + nn, err = s.r.Read(buf[n:]) + n += nn + } + if err == io.EOF { + if n < len(buf) { + err = io.ErrUnexpectedEOF + } else { + // Readers are allowed to give EOF even though the read succeeded. + // In such cases, we discard the EOF, like io.ReadFull() does. + err = nil + } + } + return err +} + +// readByte reads a single byte from the underlying stream. +func (s *Stream) readByte() (byte, error) { + if err := s.willRead(1); err != nil { + return 0, err + } + b, err := s.r.ReadByte() + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + return b, err +} + +// willRead is called before any read from the underlying stream. It checks +// n against size limits, and updates the limits if n doesn't overflow them. +func (s *Stream) willRead(n uint64) error { + s.kind = -1 // rearm Kind + + if inList, limit := s.listLimit(); inList { + if n > limit { + return ErrElemTooLarge + } + s.stack[len(s.stack)-1] = limit - n + } + if s.limited { + if n > s.remaining { + return ErrValueTooLarge + } + s.remaining -= n + } + return nil +} + +// listLimit returns the amount of data remaining in the innermost list. +func (s *Stream) listLimit() (inList bool, limit uint64) { + if len(s.stack) == 0 { + return false, 0 + } + return true, s.stack[len(s.stack)-1] +} diff --git a/encoding/rlp/decode_tail_test.go b/encoding/rlp/decode_tail_test.go new file mode 100644 index 000000000..884c1148b --- /dev/null +++ b/encoding/rlp/decode_tail_test.go @@ -0,0 +1,49 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rlp + +import ( + "bytes" + "fmt" +) + +type structWithTail struct { + A, B uint + C []uint `rlp:"tail"` +} + +func ExampleDecode_structTagTail() { + // In this example, the "tail" struct tag is used to decode lists of + // differing length into a struct. + var val structWithTail + + err := Decode(bytes.NewReader([]byte{0xC4, 0x01, 0x02, 0x03, 0x04}), &val) + fmt.Printf("with 4 elements: err=%v val=%v\n", err, val) + + err = Decode(bytes.NewReader([]byte{0xC6, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06}), &val) + fmt.Printf("with 6 elements: err=%v val=%v\n", err, val) + + // Note that at least two list elements must be present to + // fill fields A and B: + err = Decode(bytes.NewReader([]byte{0xC1, 0x01}), &val) + fmt.Printf("with 1 element: err=%q\n", err) + + // Output: + // with 4 elements: err= val={1 2 [3 4]} + // with 6 elements: err= val={1 2 [3 4 5 6]} + // with 1 element: err="rlp: too few elements for rlp.structWithTail" +} diff --git a/encoding/rlp/decode_test.go b/encoding/rlp/decode_test.go new file mode 100644 index 000000000..24b3cf986 --- /dev/null +++ b/encoding/rlp/decode_test.go @@ -0,0 +1,1211 @@ +// Copyright 2014 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rlp + +import ( + "bytes" + "encoding/hex" + "errors" + "fmt" + "io" + "math/big" + "reflect" + "strings" + "testing" + + "github.com/onflow/flow-go-sdk/encoding/common/math" +) + +func TestStreamKind(t *testing.T) { + tests := []struct { + input string + wantKind Kind + wantLen uint64 + }{ + {"00", Byte, 0}, + {"01", Byte, 0}, + {"7F", Byte, 0}, + {"80", String, 0}, + {"B7", String, 55}, + {"B90400", String, 1024}, + {"BFFFFFFFFFFFFFFFFF", String, ^uint64(0)}, + {"C0", List, 0}, + {"C8", List, 8}, + {"F7", List, 55}, + {"F90400", List, 1024}, + {"FFFFFFFFFFFFFFFFFF", List, ^uint64(0)}, + } + + for i, test := range tests { + // using plainReader to inhibit input limit errors. + s := NewStream(newPlainReader(unhex(test.input)), 0) + kind, len, err := s.Kind() + if err != nil { + t.Errorf("test %d: Kind returned error: %v", i, err) + continue + } + if kind != test.wantKind { + t.Errorf("test %d: kind mismatch: got %d, want %d", i, kind, test.wantKind) + } + if len != test.wantLen { + t.Errorf("test %d: len mismatch: got %d, want %d", i, len, test.wantLen) + } + } +} + +func TestNewListStream(t *testing.T) { + ls := NewListStream(bytes.NewReader(unhex("0101010101")), 3) + if k, size, err := ls.Kind(); k != List || size != 3 || err != nil { + t.Errorf("Kind() returned (%v, %d, %v), expected (List, 3, nil)", k, size, err) + } + if size, err := ls.List(); size != 3 || err != nil { + t.Errorf("List() returned (%d, %v), expected (3, nil)", size, err) + } + for i := 0; i < 3; i++ { + if val, err := ls.Uint(); val != 1 || err != nil { + t.Errorf("Uint() returned (%d, %v), expected (1, nil)", val, err) + } + } + if err := ls.ListEnd(); err != nil { + t.Errorf("ListEnd() returned %v, expected (3, nil)", err) + } +} + +func TestStreamErrors(t *testing.T) { + withoutInputLimit := func(b []byte) *Stream { + return NewStream(newPlainReader(b), 0) + } + withCustomInputLimit := func(limit uint64) func([]byte) *Stream { + return func(b []byte) *Stream { + return NewStream(bytes.NewReader(b), limit) + } + } + + type calls []string + tests := []struct { + string + calls + newStream func([]byte) *Stream // uses bytes.Reader if nil + error error + }{ + {"C0", calls{"Bytes"}, nil, ErrExpectedString}, + {"C0", calls{"Uint"}, nil, ErrExpectedString}, + {"89000000000000000001", calls{"Uint"}, nil, errUintOverflow}, + {"00", calls{"List"}, nil, ErrExpectedList}, + {"80", calls{"List"}, nil, ErrExpectedList}, + {"C0", calls{"List", "Uint"}, nil, EOL}, + {"C8C9010101010101010101", calls{"List", "Kind"}, nil, ErrElemTooLarge}, + {"C3C2010201", calls{"List", "List", "Uint", "Uint", "ListEnd", "Uint"}, nil, EOL}, + {"00", calls{"ListEnd"}, nil, errNotInList}, + {"C401020304", calls{"List", "Uint", "ListEnd"}, nil, errNotAtEOL}, + + // Non-canonical integers (e.g. leading zero bytes). + {"00", calls{"Uint"}, nil, ErrCanonInt}, + {"820002", calls{"Uint"}, nil, ErrCanonInt}, + {"8133", calls{"Uint"}, nil, ErrCanonSize}, + {"817F", calls{"Uint"}, nil, ErrCanonSize}, + {"8180", calls{"Uint"}, nil, nil}, + + // Non-valid boolean + {"02", calls{"Bool"}, nil, errors.New("rlp: invalid boolean value: 2")}, + + // Size tags must use the smallest possible encoding. + // Leading zero bytes in the size tag are also rejected. + {"8100", calls{"Uint"}, nil, ErrCanonSize}, + {"8100", calls{"Bytes"}, nil, ErrCanonSize}, + {"8101", calls{"Bytes"}, nil, ErrCanonSize}, + {"817F", calls{"Bytes"}, nil, ErrCanonSize}, + {"8180", calls{"Bytes"}, nil, nil}, + {"B800", calls{"Kind"}, withoutInputLimit, ErrCanonSize}, + {"B90000", calls{"Kind"}, withoutInputLimit, ErrCanonSize}, + {"B90055", calls{"Kind"}, withoutInputLimit, ErrCanonSize}, + {"BA0002FFFF", calls{"Bytes"}, withoutInputLimit, ErrCanonSize}, + {"F800", calls{"Kind"}, withoutInputLimit, ErrCanonSize}, + {"F90000", calls{"Kind"}, withoutInputLimit, ErrCanonSize}, + {"F90055", calls{"Kind"}, withoutInputLimit, ErrCanonSize}, + {"FA0002FFFF", calls{"List"}, withoutInputLimit, ErrCanonSize}, + + // Expected EOF + {"", calls{"Kind"}, nil, io.EOF}, + {"", calls{"Uint"}, nil, io.EOF}, + {"", calls{"List"}, nil, io.EOF}, + {"8180", calls{"Uint", "Uint"}, nil, io.EOF}, + {"C0", calls{"List", "ListEnd", "List"}, nil, io.EOF}, + + {"", calls{"List"}, withoutInputLimit, io.EOF}, + {"8180", calls{"Uint", "Uint"}, withoutInputLimit, io.EOF}, + {"C0", calls{"List", "ListEnd", "List"}, withoutInputLimit, io.EOF}, + + // Input limit errors. + {"81", calls{"Bytes"}, nil, ErrValueTooLarge}, + {"81", calls{"Uint"}, nil, ErrValueTooLarge}, + {"81", calls{"Raw"}, nil, ErrValueTooLarge}, + {"BFFFFFFFFFFFFFFFFFFF", calls{"Bytes"}, nil, ErrValueTooLarge}, + {"C801", calls{"List"}, nil, ErrValueTooLarge}, + + // Test for list element size check overflow. + {"CD04040404FFFFFFFFFFFFFFFFFF0303", calls{"List", "Uint", "Uint", "Uint", "Uint", "List"}, nil, ErrElemTooLarge}, + + // Test for input limit overflow. Since we are counting the limit + // down toward zero in Stream.remaining, reading too far can overflow + // remaining to a large value, effectively disabling the limit. + {"C40102030401", calls{"Raw", "Uint"}, withCustomInputLimit(5), io.EOF}, + {"C4010203048180", calls{"Raw", "Uint"}, withCustomInputLimit(6), ErrValueTooLarge}, + + // Check that the same calls are fine without a limit. + {"C40102030401", calls{"Raw", "Uint"}, withoutInputLimit, nil}, + {"C4010203048180", calls{"Raw", "Uint"}, withoutInputLimit, nil}, + + // Unexpected EOF. This only happens when there is + // no input limit, so the reader needs to be 'dumbed down'. + {"81", calls{"Bytes"}, withoutInputLimit, io.ErrUnexpectedEOF}, + {"81", calls{"Uint"}, withoutInputLimit, io.ErrUnexpectedEOF}, + {"BFFFFFFFFFFFFFFF", calls{"Bytes"}, withoutInputLimit, io.ErrUnexpectedEOF}, + {"C801", calls{"List", "Uint", "Uint"}, withoutInputLimit, io.ErrUnexpectedEOF}, + + // This test verifies that the input position is advanced + // correctly when calling Bytes for empty strings. Kind can be called + // any number of times in between and doesn't advance. + {"C3808080", calls{ + "List", // enter the list + "Bytes", // past first element + + "Kind", "Kind", "Kind", // this shouldn't advance + + "Bytes", // past second element + + "Kind", "Kind", // can't hurt to try + + "Bytes", // past final element + "Bytes", // this one should fail + }, nil, EOL}, + } + +testfor: + for i, test := range tests { + if test.newStream == nil { + test.newStream = func(b []byte) *Stream { return NewStream(bytes.NewReader(b), 0) } + } + s := test.newStream(unhex(test.string)) + rs := reflect.ValueOf(s) + for j, call := range test.calls { + fval := rs.MethodByName(call) + ret := fval.Call(nil) + err := "" + if lastret := ret[len(ret)-1].Interface(); lastret != nil { + err = lastret.(error).Error() + } + if j == len(test.calls)-1 { + want := "" + if test.error != nil { + want = test.error.Error() + } + if err != want { + t.Log(test) + t.Errorf("test %d: last call (%s) error mismatch\ngot: %s\nwant: %s", + i, call, err, test.error) + } + } else if err != "" { + t.Log(test) + t.Errorf("test %d: call %d (%s) unexpected error: %q", i, j, call, err) + continue testfor + } + } + } +} + +func TestStreamList(t *testing.T) { + s := NewStream(bytes.NewReader(unhex("C80102030405060708")), 0) + + len, err := s.List() + if err != nil { + t.Fatalf("List error: %v", err) + } + if len != 8 { + t.Fatalf("List returned invalid length, got %d, want 8", len) + } + + for i := uint64(1); i <= 8; i++ { + v, err := s.Uint() + if err != nil { + t.Fatalf("Uint error: %v", err) + } + if i != v { + t.Errorf("Uint returned wrong value, got %d, want %d", v, i) + } + } + + if _, err := s.Uint(); err != EOL { + t.Errorf("Uint error mismatch, got %v, want %v", err, EOL) + } + if err = s.ListEnd(); err != nil { + t.Fatalf("ListEnd error: %v", err) + } +} + +func TestStreamRaw(t *testing.T) { + tests := []struct { + input string + output string + }{ + { + "C58401010101", + "8401010101", + }, + { + "F842B84001010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101", + "B84001010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101", + }, + } + for i, tt := range tests { + s := NewStream(bytes.NewReader(unhex(tt.input)), 0) + s.List() + + want := unhex(tt.output) + raw, err := s.Raw() + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(want, raw) { + t.Errorf("test %d: raw mismatch: got %x, want %x", i, raw, want) + } + } +} + +func TestStreamReadBytes(t *testing.T) { + tests := []struct { + input string + size int + err string + }{ + // kind List + {input: "C0", size: 1, err: "rlp: expected String or Byte"}, + // kind Byte + {input: "04", size: 0, err: "input value has wrong size 1, want 0"}, + {input: "04", size: 1}, + {input: "04", size: 2, err: "input value has wrong size 1, want 2"}, + // kind String + {input: "820102", size: 0, err: "input value has wrong size 2, want 0"}, + {input: "820102", size: 1, err: "input value has wrong size 2, want 1"}, + {input: "820102", size: 2}, + {input: "820102", size: 3, err: "input value has wrong size 2, want 3"}, + } + + for _, test := range tests { + test := test + name := fmt.Sprintf("input_%s/size_%d", test.input, test.size) + t.Run(name, func(t *testing.T) { + s := NewStream(bytes.NewReader(unhex(test.input)), 0) + b := make([]byte, test.size) + err := s.ReadBytes(b) + if test.err == "" { + if err != nil { + t.Errorf("unexpected error %q", err) + } + } else { + if err == nil { + t.Errorf("expected error, got nil") + } else if err.Error() != test.err { + t.Errorf("wrong error %q", err) + } + } + }) + } +} + +func TestDecodeErrors(t *testing.T) { + r := bytes.NewReader(nil) + + if err := Decode(r, nil); err != errDecodeIntoNil { + t.Errorf("Decode(r, nil) error mismatch, got %q, want %q", err, errDecodeIntoNil) + } + + var nilptr *struct{} + if err := Decode(r, nilptr); err != errDecodeIntoNil { + t.Errorf("Decode(r, nilptr) error mismatch, got %q, want %q", err, errDecodeIntoNil) + } + + if err := Decode(r, struct{}{}); err != errNoPointer { + t.Errorf("Decode(r, struct{}{}) error mismatch, got %q, want %q", err, errNoPointer) + } + + expectErr := "rlp: type chan bool is not RLP-serializable" + if err := Decode(r, new(chan bool)); err == nil || err.Error() != expectErr { + t.Errorf("Decode(r, new(chan bool)) error mismatch, got %q, want %q", err, expectErr) + } + + if err := Decode(r, new(uint)); err != io.EOF { + t.Errorf("Decode(r, new(int)) error mismatch, got %q, want %q", err, io.EOF) + } +} + +type decodeTest struct { + input string + ptr interface{} + value interface{} + error string +} + +type simplestruct struct { + A uint + B string +} + +type recstruct struct { + I uint + Child *recstruct `rlp:"nil"` +} + +type bigIntStruct struct { + I *big.Int + B string +} + +type invalidNilTag struct { + X []byte `rlp:"nil"` +} + +type invalidTail1 struct { + A uint `rlp:"tail"` + B string +} + +type invalidTail2 struct { + A uint + B string `rlp:"tail"` +} + +type tailRaw struct { + A uint + Tail []RawValue `rlp:"tail"` +} + +type tailUint struct { + A uint + Tail []uint `rlp:"tail"` +} + +type tailPrivateFields struct { + A uint + Tail []uint `rlp:"tail"` + x, y bool //lint:ignore U1000 unused fields required for testing purposes. +} + +type nilListUint struct { + X *uint `rlp:"nilList"` +} + +type nilStringSlice struct { + X *[]uint `rlp:"nilString"` +} + +type intField struct { + X int +} + +type optionalFields struct { + A uint + B uint `rlp:"optional"` + C uint `rlp:"optional"` +} + +type optionalAndTailField struct { + A uint + B uint `rlp:"optional"` + Tail []uint `rlp:"tail"` +} + +type optionalBigIntField struct { + A uint + B *big.Int `rlp:"optional"` +} + +type optionalPtrField struct { + A uint + B *[3]byte `rlp:"optional"` +} + +type optionalPtrFieldNil struct { + A uint + B *[3]byte `rlp:"optional,nil"` +} + +type ignoredField struct { + A uint + B uint `rlp:"-"` + C uint +} + +var ( + veryBigInt = new(big.Int).Add( + big.NewInt(0).Lsh(big.NewInt(0xFFFFFFFFFFFFFF), 16), + big.NewInt(0xFFFF), + ) + veryVeryBigInt = new(big.Int).Exp(veryBigInt, big.NewInt(8), nil) +) + +var decodeTests = []decodeTest{ + // booleans + {input: "01", ptr: new(bool), value: true}, + {input: "80", ptr: new(bool), value: false}, + {input: "02", ptr: new(bool), error: "rlp: invalid boolean value: 2"}, + + // integers + {input: "05", ptr: new(uint32), value: uint32(5)}, + {input: "80", ptr: new(uint32), value: uint32(0)}, + {input: "820505", ptr: new(uint32), value: uint32(0x0505)}, + {input: "83050505", ptr: new(uint32), value: uint32(0x050505)}, + {input: "8405050505", ptr: new(uint32), value: uint32(0x05050505)}, + {input: "850505050505", ptr: new(uint32), error: "rlp: input string too long for uint32"}, + {input: "C0", ptr: new(uint32), error: "rlp: expected input string or byte for uint32"}, + {input: "00", ptr: new(uint32), error: "rlp: non-canonical integer (leading zero bytes) for uint32"}, + {input: "8105", ptr: new(uint32), error: "rlp: non-canonical size information for uint32"}, + {input: "820004", ptr: new(uint32), error: "rlp: non-canonical integer (leading zero bytes) for uint32"}, + {input: "B8020004", ptr: new(uint32), error: "rlp: non-canonical size information for uint32"}, + + // slices + {input: "C0", ptr: new([]uint), value: []uint{}}, + {input: "C80102030405060708", ptr: new([]uint), value: []uint{1, 2, 3, 4, 5, 6, 7, 8}}, + {input: "F8020004", ptr: new([]uint), error: "rlp: non-canonical size information for []uint"}, + + // arrays + {input: "C50102030405", ptr: new([5]uint), value: [5]uint{1, 2, 3, 4, 5}}, + {input: "C0", ptr: new([5]uint), error: "rlp: input list has too few elements for [5]uint"}, + {input: "C102", ptr: new([5]uint), error: "rlp: input list has too few elements for [5]uint"}, + {input: "C6010203040506", ptr: new([5]uint), error: "rlp: input list has too many elements for [5]uint"}, + {input: "F8020004", ptr: new([5]uint), error: "rlp: non-canonical size information for [5]uint"}, + + // zero sized arrays + {input: "C0", ptr: new([0]uint), value: [0]uint{}}, + {input: "C101", ptr: new([0]uint), error: "rlp: input list has too many elements for [0]uint"}, + + // byte slices + {input: "01", ptr: new([]byte), value: []byte{1}}, + {input: "80", ptr: new([]byte), value: []byte{}}, + {input: "8D6162636465666768696A6B6C6D", ptr: new([]byte), value: []byte("abcdefghijklm")}, + {input: "C0", ptr: new([]byte), error: "rlp: expected input string or byte for []uint8"}, + {input: "8105", ptr: new([]byte), error: "rlp: non-canonical size information for []uint8"}, + + // byte arrays + {input: "02", ptr: new([1]byte), value: [1]byte{2}}, + {input: "8180", ptr: new([1]byte), value: [1]byte{128}}, + {input: "850102030405", ptr: new([5]byte), value: [5]byte{1, 2, 3, 4, 5}}, + + // byte array errors + {input: "02", ptr: new([5]byte), error: "rlp: input string too short for [5]uint8"}, + {input: "80", ptr: new([5]byte), error: "rlp: input string too short for [5]uint8"}, + {input: "820000", ptr: new([5]byte), error: "rlp: input string too short for [5]uint8"}, + {input: "C0", ptr: new([5]byte), error: "rlp: expected input string or byte for [5]uint8"}, + {input: "C3010203", ptr: new([5]byte), error: "rlp: expected input string or byte for [5]uint8"}, + {input: "86010203040506", ptr: new([5]byte), error: "rlp: input string too long for [5]uint8"}, + {input: "8105", ptr: new([1]byte), error: "rlp: non-canonical size information for [1]uint8"}, + {input: "817F", ptr: new([1]byte), error: "rlp: non-canonical size information for [1]uint8"}, + + // zero sized byte arrays + {input: "80", ptr: new([0]byte), value: [0]byte{}}, + {input: "01", ptr: new([0]byte), error: "rlp: input string too long for [0]uint8"}, + {input: "8101", ptr: new([0]byte), error: "rlp: input string too long for [0]uint8"}, + + // strings + {input: "00", ptr: new(string), value: "\000"}, + {input: "8D6162636465666768696A6B6C6D", ptr: new(string), value: "abcdefghijklm"}, + {input: "C0", ptr: new(string), error: "rlp: expected input string or byte for string"}, + + // big ints + {input: "80", ptr: new(*big.Int), value: big.NewInt(0)}, + {input: "01", ptr: new(*big.Int), value: big.NewInt(1)}, + {input: "89FFFFFFFFFFFFFFFFFF", ptr: new(*big.Int), value: veryBigInt}, + {input: "B848FFFFFFFFFFFFFFFFF800000000000000001BFFFFFFFFFFFFFFFFC8000000000000000045FFFFFFFFFFFFFFFFC800000000000000001BFFFFFFFFFFFFFFFFF8000000000000000001", ptr: new(*big.Int), value: veryVeryBigInt}, + {input: "10", ptr: new(big.Int), value: *big.NewInt(16)}, // non-pointer also works + {input: "C0", ptr: new(*big.Int), error: "rlp: expected input string or byte for *big.Int"}, + {input: "00", ptr: new(*big.Int), error: "rlp: non-canonical integer (leading zero bytes) for *big.Int"}, + {input: "820001", ptr: new(*big.Int), error: "rlp: non-canonical integer (leading zero bytes) for *big.Int"}, + {input: "8105", ptr: new(*big.Int), error: "rlp: non-canonical size information for *big.Int"}, + + // structs + { + input: "C50583343434", + ptr: new(simplestruct), + value: simplestruct{5, "444"}, + }, + { + input: "C601C402C203C0", + ptr: new(recstruct), + value: recstruct{1, &recstruct{2, &recstruct{3, nil}}}, + }, + { + // This checks that empty big.Int works correctly in struct context. It's easy to + // miss the update of s.kind for this case, so it needs its own test. + input: "C58083343434", + ptr: new(bigIntStruct), + value: bigIntStruct{new(big.Int), "444"}, + }, + + // struct errors + { + input: "C0", + ptr: new(simplestruct), + error: "rlp: too few elements for rlp.simplestruct", + }, + { + input: "C105", + ptr: new(simplestruct), + error: "rlp: too few elements for rlp.simplestruct", + }, + { + input: "C7C50583343434C0", + ptr: new([]*simplestruct), + error: "rlp: too few elements for rlp.simplestruct, decoding into ([]*rlp.simplestruct)[1]", + }, + { + input: "83222222", + ptr: new(simplestruct), + error: "rlp: expected input list for rlp.simplestruct", + }, + { + input: "C3010101", + ptr: new(simplestruct), + error: "rlp: input list has too many elements for rlp.simplestruct", + }, + { + input: "C501C3C00000", + ptr: new(recstruct), + error: "rlp: expected input string or byte for uint, decoding into (rlp.recstruct).Child.I", + }, + { + input: "C103", + ptr: new(intField), + error: "rlp: type int is not RLP-serializable (struct field rlp.intField.X)", + }, + { + input: "C50102C20102", + ptr: new(tailUint), + error: "rlp: expected input string or byte for uint, decoding into (rlp.tailUint).Tail[1]", + }, + { + input: "C0", + ptr: new(invalidNilTag), + error: `rlp: invalid struct tag "nil" for rlp.invalidNilTag.X (field is not a pointer)`, + }, + + // struct tag "tail" + { + input: "C3010203", + ptr: new(tailRaw), + value: tailRaw{A: 1, Tail: []RawValue{unhex("02"), unhex("03")}}, + }, + { + input: "C20102", + ptr: new(tailRaw), + value: tailRaw{A: 1, Tail: []RawValue{unhex("02")}}, + }, + { + input: "C101", + ptr: new(tailRaw), + value: tailRaw{A: 1, Tail: []RawValue{}}, + }, + { + input: "C3010203", + ptr: new(tailPrivateFields), + value: tailPrivateFields{A: 1, Tail: []uint{2, 3}}, + }, + { + input: "C0", + ptr: new(invalidTail1), + error: `rlp: invalid struct tag "tail" for rlp.invalidTail1.A (must be on last field)`, + }, + { + input: "C0", + ptr: new(invalidTail2), + error: `rlp: invalid struct tag "tail" for rlp.invalidTail2.B (field type is not slice)`, + }, + + // struct tag "-" + { + input: "C20102", + ptr: new(ignoredField), + value: ignoredField{A: 1, C: 2}, + }, + + // struct tag "nilList" + { + input: "C180", + ptr: new(nilListUint), + error: "rlp: wrong kind of empty value (got String, want List) for *uint, decoding into (rlp.nilListUint).X", + }, + { + input: "C1C0", + ptr: new(nilListUint), + value: nilListUint{}, + }, + { + input: "C103", + ptr: new(nilListUint), + value: func() interface{} { + v := uint(3) + return nilListUint{X: &v} + }(), + }, + + // struct tag "nilString" + { + input: "C1C0", + ptr: new(nilStringSlice), + error: "rlp: wrong kind of empty value (got List, want String) for *[]uint, decoding into (rlp.nilStringSlice).X", + }, + { + input: "C180", + ptr: new(nilStringSlice), + value: nilStringSlice{}, + }, + { + input: "C2C103", + ptr: new(nilStringSlice), + value: nilStringSlice{X: &[]uint{3}}, + }, + + // struct tag "optional" + { + input: "C101", + ptr: new(optionalFields), + value: optionalFields{1, 0, 0}, + }, + { + input: "C20102", + ptr: new(optionalFields), + value: optionalFields{1, 2, 0}, + }, + { + input: "C3010203", + ptr: new(optionalFields), + value: optionalFields{1, 2, 3}, + }, + { + input: "C401020304", + ptr: new(optionalFields), + error: "rlp: input list has too many elements for rlp.optionalFields", + }, + { + input: "C101", + ptr: new(optionalAndTailField), + value: optionalAndTailField{A: 1}, + }, + { + input: "C20102", + ptr: new(optionalAndTailField), + value: optionalAndTailField{A: 1, B: 2, Tail: []uint{}}, + }, + { + input: "C401020304", + ptr: new(optionalAndTailField), + value: optionalAndTailField{A: 1, B: 2, Tail: []uint{3, 4}}, + }, + { + input: "C101", + ptr: new(optionalBigIntField), + value: optionalBigIntField{A: 1, B: nil}, + }, + { + input: "C20102", + ptr: new(optionalBigIntField), + value: optionalBigIntField{A: 1, B: big.NewInt(2)}, + }, + { + input: "C101", + ptr: new(optionalPtrField), + value: optionalPtrField{A: 1}, + }, + { + input: "C20180", // not accepted because "optional" doesn't enable "nil" + ptr: new(optionalPtrField), + error: "rlp: input string too short for [3]uint8, decoding into (rlp.optionalPtrField).B", + }, + { + input: "C20102", + ptr: new(optionalPtrField), + error: "rlp: input string too short for [3]uint8, decoding into (rlp.optionalPtrField).B", + }, + { + input: "C50183010203", + ptr: new(optionalPtrField), + value: optionalPtrField{A: 1, B: &[3]byte{1, 2, 3}}, + }, + { + input: "C101", + ptr: new(optionalPtrFieldNil), + value: optionalPtrFieldNil{A: 1}, + }, + { + input: "C20180", // accepted because "nil" tag allows empty input + ptr: new(optionalPtrFieldNil), + value: optionalPtrFieldNil{A: 1}, + }, + { + input: "C20102", + ptr: new(optionalPtrFieldNil), + error: "rlp: input string too short for [3]uint8, decoding into (rlp.optionalPtrFieldNil).B", + }, + + // struct tag "optional" field clearing + { + input: "C101", + ptr: &optionalFields{A: 9, B: 8, C: 7}, + value: optionalFields{A: 1, B: 0, C: 0}, + }, + { + input: "C20102", + ptr: &optionalFields{A: 9, B: 8, C: 7}, + value: optionalFields{A: 1, B: 2, C: 0}, + }, + { + input: "C20102", + ptr: &optionalAndTailField{A: 9, B: 8, Tail: []uint{7, 6, 5}}, + value: optionalAndTailField{A: 1, B: 2, Tail: []uint{}}, + }, + { + input: "C101", + ptr: &optionalPtrField{A: 9, B: &[3]byte{8, 7, 6}}, + value: optionalPtrField{A: 1}, + }, + + // RawValue + {input: "01", ptr: new(RawValue), value: RawValue(unhex("01"))}, + {input: "82FFFF", ptr: new(RawValue), value: RawValue(unhex("82FFFF"))}, + {input: "C20102", ptr: new([]RawValue), value: []RawValue{unhex("01"), unhex("02")}}, + + // pointers + {input: "00", ptr: new(*[]byte), value: &[]byte{0}}, + {input: "80", ptr: new(*uint), value: uintp(0)}, + {input: "C0", ptr: new(*uint), error: "rlp: expected input string or byte for uint"}, + {input: "07", ptr: new(*uint), value: uintp(7)}, + {input: "817F", ptr: new(*uint), error: "rlp: non-canonical size information for uint"}, + {input: "8180", ptr: new(*uint), value: uintp(0x80)}, + {input: "C109", ptr: new(*[]uint), value: &[]uint{9}}, + {input: "C58403030303", ptr: new(*[][]byte), value: &[][]byte{{3, 3, 3, 3}}}, + + // check that input position is advanced also for empty values. + {input: "C3808005", ptr: new([]*uint), value: []*uint{uintp(0), uintp(0), uintp(5)}}, + + // interface{} + {input: "00", ptr: new(interface{}), value: []byte{0}}, + {input: "01", ptr: new(interface{}), value: []byte{1}}, + {input: "80", ptr: new(interface{}), value: []byte{}}, + {input: "850505050505", ptr: new(interface{}), value: []byte{5, 5, 5, 5, 5}}, + {input: "C0", ptr: new(interface{}), value: []interface{}{}}, + {input: "C50183040404", ptr: new(interface{}), value: []interface{}{[]byte{1}, []byte{4, 4, 4}}}, + { + input: "C3010203", + ptr: new([]io.Reader), + error: "rlp: type io.Reader is not RLP-serializable", + }, + + // fuzzer crashes + { + input: "c330f9c030f93030ce3030303030303030bd303030303030", + ptr: new(interface{}), + error: "rlp: element is larger than containing list", + }, +} + +func uintp(i uint) *uint { return &i } + +func runTests(t *testing.T, decode func([]byte, interface{}) error) { + for i, test := range decodeTests { + input, err := hex.DecodeString(test.input) + if err != nil { + t.Errorf("test %d: invalid hex input %q", i, test.input) + continue + } + err = decode(input, test.ptr) + if err != nil && test.error == "" { + t.Errorf("test %d: unexpected Decode error: %v\ndecoding into %T\ninput %q", + i, err, test.ptr, test.input) + continue + } + if test.error != "" && fmt.Sprint(err) != test.error { + t.Errorf("test %d: Decode error mismatch\ngot %v\nwant %v\ndecoding into %T\ninput %q", + i, err, test.error, test.ptr, test.input) + continue + } + deref := reflect.ValueOf(test.ptr).Elem().Interface() + if err == nil && !reflect.DeepEqual(deref, test.value) { + t.Errorf("test %d: value mismatch\ngot %#v\nwant %#v\ndecoding into %T\ninput %q", + i, deref, test.value, test.ptr, test.input) + } + } +} + +func TestDecodeWithByteReader(t *testing.T) { + runTests(t, func(input []byte, into interface{}) error { + return Decode(bytes.NewReader(input), into) + }) +} + +func testDecodeWithEncReader(t *testing.T, n int) { + s := strings.Repeat("0", n) + _, r, _ := EncodeToReader(s) + var decoded string + err := Decode(r, &decoded) + if err != nil { + t.Errorf("Unexpected decode error with n=%v: %v", n, err) + } + if decoded != s { + t.Errorf("Decode mismatch with n=%v", n) + } +} + +// This is a regression test checking that decoding from encReader +// works for RLP values of size 8192 bytes or more. +func TestDecodeWithEncReader(t *testing.T) { + testDecodeWithEncReader(t, 8188) // length with header is 8191 + testDecodeWithEncReader(t, 8189) // length with header is 8192 +} + +// plainReader reads from a byte slice but does not +// implement ReadByte. It is also not recognized by the +// size validation. This is useful to test how the decoder +// behaves on a non-buffered input stream. +type plainReader []byte + +func newPlainReader(b []byte) io.Reader { + return (*plainReader)(&b) +} + +func (r *plainReader) Read(buf []byte) (n int, err error) { + if len(*r) == 0 { + return 0, io.EOF + } + n = copy(buf, *r) + *r = (*r)[n:] + return n, nil +} + +func TestDecodeWithNonByteReader(t *testing.T) { + runTests(t, func(input []byte, into interface{}) error { + return Decode(newPlainReader(input), into) + }) +} + +func TestDecodeStreamReset(t *testing.T) { + s := NewStream(nil, 0) + runTests(t, func(input []byte, into interface{}) error { + s.Reset(bytes.NewReader(input), 0) + return s.Decode(into) + }) +} + +type testDecoder struct{ called bool } + +func (t *testDecoder) DecodeRLP(s *Stream) error { + if _, err := s.Uint(); err != nil { + return err + } + t.called = true + return nil +} + +func TestDecodeDecoder(t *testing.T) { + var s struct { + T1 testDecoder + T2 *testDecoder + T3 **testDecoder + } + if err := Decode(bytes.NewReader(unhex("C3010203")), &s); err != nil { + t.Fatalf("Decode error: %v", err) + } + + if !s.T1.called { + t.Errorf("DecodeRLP was not called for (non-pointer) testDecoder") + } + + if s.T2 == nil { + t.Errorf("*testDecoder has not been allocated") + } else if !s.T2.called { + t.Errorf("DecodeRLP was not called for *testDecoder") + } + + if s.T3 == nil || *s.T3 == nil { + t.Errorf("**testDecoder has not been allocated") + } else if !(*s.T3).called { + t.Errorf("DecodeRLP was not called for **testDecoder") + } +} + +func TestDecodeDecoderNilPointer(t *testing.T) { + var s struct { + T1 *testDecoder `rlp:"nil"` + T2 *testDecoder + } + if err := Decode(bytes.NewReader(unhex("C2C002")), &s); err != nil { + t.Fatalf("Decode error: %v", err) + } + if s.T1 != nil { + t.Errorf("decoder T1 allocated for empty input (called: %v)", s.T1.called) + } + if s.T2 == nil || !s.T2.called { + t.Errorf("decoder T2 not allocated/called") + } +} + +type byteDecoder byte + +func (bd *byteDecoder) DecodeRLP(s *Stream) error { + _, err := s.Uint() + *bd = 255 + return err +} + +func (bd byteDecoder) called() bool { + return bd == 255 +} + +// This test verifies that the byte slice/byte array logic +// does not kick in for element types implementing Decoder. +func TestDecoderInByteSlice(t *testing.T) { + var slice []byteDecoder + if err := Decode(bytes.NewReader(unhex("C101")), &slice); err != nil { + t.Errorf("unexpected Decode error %v", err) + } else if !slice[0].called() { + t.Errorf("DecodeRLP not called for slice element") + } + + var array [1]byteDecoder + if err := Decode(bytes.NewReader(unhex("C101")), &array); err != nil { + t.Errorf("unexpected Decode error %v", err) + } else if !array[0].called() { + t.Errorf("DecodeRLP not called for array element") + } +} + +type unencodableDecoder func() + +func (f *unencodableDecoder) DecodeRLP(s *Stream) error { + if _, err := s.List(); err != nil { + return err + } + if err := s.ListEnd(); err != nil { + return err + } + *f = func() {} + return nil +} + +func TestDecoderFunc(t *testing.T) { + var x func() + if err := DecodeBytes([]byte{0xC0}, (*unencodableDecoder)(&x)); err != nil { + t.Fatal(err) + } + x() +} + +// This tests the validity checks for fields with struct tag "optional". +func TestInvalidOptionalField(t *testing.T) { + type ( + invalid1 struct { + A uint `rlp:"optional"` + B uint + } + invalid2 struct { + T []uint `rlp:"tail,optional"` + } + invalid3 struct { + T []uint `rlp:"optional,tail"` + } + ) + + tests := []struct { + v interface{} + err string + }{ + {v: new(invalid1), err: `rlp: invalid struct tag "" for rlp.invalid1.B (must be optional because preceding field "A" is optional)`}, + {v: new(invalid2), err: `rlp: invalid struct tag "optional" for rlp.invalid2.T (also has "tail" tag)`}, + {v: new(invalid3), err: `rlp: invalid struct tag "tail" for rlp.invalid3.T (also has "optional" tag)`}, + } + for _, test := range tests { + err := DecodeBytes(unhex("C20102"), test.v) + if err == nil { + t.Errorf("no error for %T", test.v) + } else if err.Error() != test.err { + t.Errorf("wrong error for %T: %v", test.v, err.Error()) + } + } + +} + +func ExampleDecode() { + input, _ := hex.DecodeString("C90A1486666F6F626172") + + type example struct { + A, B uint + String string + } + + var s example + err := Decode(bytes.NewReader(input), &s) + if err != nil { + fmt.Printf("Error: %v\n", err) + } else { + fmt.Printf("Decoded value: %#v\n", s) + } + // Output: + // Decoded value: rlp.example{A:0xa, B:0x14, String:"foobar"} +} + +func ExampleDecode_structTagNil() { + // In this example, we'll use the "nil" struct tag to change + // how a pointer-typed field is decoded. The input contains an RLP + // list of one element, an empty string. + input := []byte{0xC1, 0x80} + + // This type uses the normal rules. + // The empty input string is decoded as a pointer to an empty Go string. + var normalRules struct { + String *string + } + Decode(bytes.NewReader(input), &normalRules) + fmt.Printf("normal: String = %q\n", *normalRules.String) + + // This type uses the struct tag. + // The empty input string is decoded as a nil pointer. + var withEmptyOK struct { + String *string `rlp:"nil"` + } + Decode(bytes.NewReader(input), &withEmptyOK) + fmt.Printf("with nil tag: String = %v\n", withEmptyOK.String) + + // Output: + // normal: String = "" + // with nil tag: String = +} + +func ExampleStream() { + input, _ := hex.DecodeString("C90A1486666F6F626172") + s := NewStream(bytes.NewReader(input), 0) + + // Check what kind of value lies ahead + kind, size, _ := s.Kind() + fmt.Printf("Kind: %v size:%d\n", kind, size) + + // Enter the list + if _, err := s.List(); err != nil { + fmt.Printf("List error: %v\n", err) + return + } + + // Decode elements + fmt.Println(s.Uint()) + fmt.Println(s.Uint()) + fmt.Println(s.Bytes()) + + // Acknowledge end of list + if err := s.ListEnd(); err != nil { + fmt.Printf("ListEnd error: %v\n", err) + } + // Output: + // Kind: List size:9 + // 10 + // 20 + // [102 111 111 98 97 114] +} + +func BenchmarkDecodeUints(b *testing.B) { + enc := encodeTestSlice(90000) + b.SetBytes(int64(len(enc))) + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + var s []uint + r := bytes.NewReader(enc) + if err := Decode(r, &s); err != nil { + b.Fatalf("Decode error: %v", err) + } + } +} + +func BenchmarkDecodeUintsReused(b *testing.B) { + enc := encodeTestSlice(100000) + b.SetBytes(int64(len(enc))) + b.ReportAllocs() + b.ResetTimer() + + var s []uint + for i := 0; i < b.N; i++ { + r := bytes.NewReader(enc) + if err := Decode(r, &s); err != nil { + b.Fatalf("Decode error: %v", err) + } + } +} + +func BenchmarkDecodeByteArrayStruct(b *testing.B) { + enc, err := EncodeToBytes(&byteArrayStruct{}) + if err != nil { + b.Fatal(err) + } + b.SetBytes(int64(len(enc))) + b.ReportAllocs() + b.ResetTimer() + + var out byteArrayStruct + for i := 0; i < b.N; i++ { + if err := DecodeBytes(enc, &out); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkDecodeBigInts(b *testing.B) { + ints := make([]*big.Int, 200) + for i := range ints { + ints[i] = math.BigPow(2, int64(i)) + } + enc, err := EncodeToBytes(ints) + if err != nil { + b.Fatal(err) + } + b.SetBytes(int64(len(enc))) + b.ReportAllocs() + b.ResetTimer() + + var out []*big.Int + for i := 0; i < b.N; i++ { + if err := DecodeBytes(enc, &out); err != nil { + b.Fatal(err) + } + } +} + +func encodeTestSlice(n uint) []byte { + s := make([]uint, n) + for i := uint(0); i < n; i++ { + s[i] = i + } + b, err := EncodeToBytes(s) + if err != nil { + panic(fmt.Sprintf("encode error: %v", err)) + } + return b +} + +func unhex(str string) []byte { + b, err := hex.DecodeString(strings.ReplaceAll(str, " ", "")) + if err != nil { + panic(fmt.Sprintf("invalid hex string: %q", str)) + } + return b +} diff --git a/encoding/rlp/doc.go b/encoding/rlp/doc.go new file mode 100644 index 000000000..e4404c978 --- /dev/null +++ b/encoding/rlp/doc.go @@ -0,0 +1,161 @@ +// Copyright 2014 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +/* +Package rlp implements the RLP serialization format. + +The purpose of RLP (Recursive Linear Prefix) is to encode arbitrarily nested arrays of +binary data, and RLP is the main encoding method used to serialize objects in Ethereum. +The only purpose of RLP is to encode structure; encoding specific atomic data types (eg. +strings, ints, floats) is left up to higher-order protocols. In Ethereum integers must be +represented in big endian binary form with no leading zeroes (thus making the integer +value zero equivalent to the empty string). + +RLP values are distinguished by a type tag. The type tag precedes the value in the input +stream and defines the size and kind of the bytes that follow. + + +Encoding Rules + +Package rlp uses reflection and encodes RLP based on the Go type of the value. + +If the type implements the Encoder interface, Encode calls EncodeRLP. It does not +call EncodeRLP on nil pointer values. + +To encode a pointer, the value being pointed to is encoded. A nil pointer to a struct +type, slice or array always encodes as an empty RLP list unless the slice or array has +element type byte. A nil pointer to any other value encodes as the empty string. + +Struct values are encoded as an RLP list of all their encoded public fields. Recursive +struct types are supported. + +To encode slices and arrays, the elements are encoded as an RLP list of the value's +elements. Note that arrays and slices with element type uint8 or byte are always encoded +as an RLP string. + +A Go string is encoded as an RLP string. + +An unsigned integer value is encoded as an RLP string. Zero always encodes as an empty RLP +string. big.Int values are treated as integers. Signed integers (int, int8, int16, ...) +are not supported and will return an error when encoding. + +Boolean values are encoded as the unsigned integers zero (false) and one (true). + +An interface value encodes as the value contained in the interface. + +Floating point numbers, maps, channels and functions are not supported. + + +Decoding Rules + +Decoding uses the following type-dependent rules: + +If the type implements the Decoder interface, DecodeRLP is called. + +To decode into a pointer, the value will be decoded as the element type of the pointer. If +the pointer is nil, a new value of the pointer's element type is allocated. If the pointer +is non-nil, the existing value will be reused. Note that package rlp never leaves a +pointer-type struct field as nil unless one of the "nil" struct tags is present. + +To decode into a struct, decoding expects the input to be an RLP list. The decoded +elements of the list are assigned to each public field in the order given by the struct's +definition. The input list must contain an element for each decoded field. Decoding +returns an error if there are too few or too many elements for the struct. + +To decode into a slice, the input must be a list and the resulting slice will contain the +input elements in order. For byte slices, the input must be an RLP string. Array types +decode similarly, with the additional restriction that the number of input elements (or +bytes) must match the array's defined length. + +To decode into a Go string, the input must be an RLP string. The input bytes are taken +as-is and will not necessarily be valid UTF-8. + +To decode into an unsigned integer type, the input must also be an RLP string. The bytes +are interpreted as a big endian representation of the integer. If the RLP string is larger +than the bit size of the type, decoding will return an error. Decode also supports +*big.Int. There is no size limit for big integers. + +To decode into a boolean, the input must contain an unsigned integer of value zero (false) +or one (true). + +To decode into an interface value, one of these types is stored in the value: + + []interface{}, for RLP lists + []byte, for RLP strings + +Non-empty interface types are not supported when decoding. +Signed integers, floating point numbers, maps, channels and functions cannot be decoded into. + + +Struct Tags + +As with other encoding packages, the "-" tag ignores fields. + + type StructWithIgnoredField struct{ + Ignored uint `rlp:"-"` + Field uint + } + +Go struct values encode/decode as RLP lists. There are two ways of influencing the mapping +of fields to list elements. The "tail" tag, which may only be used on the last exported +struct field, allows slurping up any excess list elements into a slice. + + type StructWithTail struct{ + Field uint + Tail []string `rlp:"tail"` + } + +The "optional" tag says that the field may be omitted if it is zero-valued. If this tag is +used on a struct field, all subsequent public fields must also be declared optional. + +When encoding a struct with optional fields, the output RLP list contains all values up to +the last non-zero optional field. + +When decoding into a struct, optional fields may be omitted from the end of the input +list. For the example below, this means input lists of one, two, or three elements are +accepted. + + type StructWithOptionalFields struct{ + Required uint + Optional1 uint `rlp:"optional"` + Optional2 uint `rlp:"optional"` + } + +The "nil", "nilList" and "nilString" tags apply to pointer-typed fields only, and change +the decoding rules for the field type. For regular pointer fields without the "nil" tag, +input values must always match the required input length exactly and the decoder does not +produce nil values. When the "nil" tag is set, input values of size zero decode as a nil +pointer. This is especially useful for recursive types. + + type StructWithNilField struct { + Field *[3]byte `rlp:"nil"` + } + +In the example above, Field allows two possible input sizes. For input 0xC180 (a list +containing an empty string) Field is set to nil after decoding. For input 0xC483000000 (a +list containing a 3-byte string), Field is set to a non-nil array pointer. + +RLP supports two kinds of empty values: empty lists and empty strings. When using the +"nil" tag, the kind of empty value allowed for a type is chosen automatically. A field +whose Go type is a pointer to an unsigned integer, string, boolean or byte array/slice +expects an empty RLP string. Any other pointer field type encodes/decodes as an empty RLP +list. + +The choice of null value can be made explicit with the "nilList" and "nilString" struct +tags. Using these tags encodes/decodes a Go nil pointer value as the empty RLP value kind +defined by the tag. +*/ +package rlp diff --git a/encoding/rlp/encbuffer.go b/encoding/rlp/encbuffer.go new file mode 100644 index 000000000..687949c04 --- /dev/null +++ b/encoding/rlp/encbuffer.go @@ -0,0 +1,398 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rlp + +import ( + "io" + "math/big" + "reflect" + "sync" +) + +type encBuffer struct { + str []byte // string data, contains everything except list headers + lheads []listhead // all list headers + lhsize int // sum of sizes of all encoded list headers + sizebuf [9]byte // auxiliary buffer for uint encoding +} + +// The global encBuffer pool. +var encBufferPool = sync.Pool{ + New: func() interface{} { return new(encBuffer) }, +} + +func getEncBuffer() *encBuffer { + buf := encBufferPool.Get().(*encBuffer) + buf.reset() + return buf +} + +func (buf *encBuffer) reset() { + buf.lhsize = 0 + buf.str = buf.str[:0] + buf.lheads = buf.lheads[:0] +} + +// size returns the length of the encoded data. +func (buf *encBuffer) size() int { + return len(buf.str) + buf.lhsize +} + +// makeBytes creates the encoder output. +func (w *encBuffer) makeBytes() []byte { + out := make([]byte, w.size()) + w.copyTo(out) + return out +} + +func (w *encBuffer) copyTo(dst []byte) { + strpos := 0 + pos := 0 + for _, head := range w.lheads { + // write string data before header + n := copy(dst[pos:], w.str[strpos:head.offset]) + pos += n + strpos += n + // write the header + enc := head.encode(dst[pos:]) + pos += len(enc) + } + // copy string data after the last list header + copy(dst[pos:], w.str[strpos:]) +} + +// writeTo writes the encoder output to w. +func (buf *encBuffer) writeTo(w io.Writer) (err error) { + strpos := 0 + for _, head := range buf.lheads { + // write string data before header + if head.offset-strpos > 0 { + n, err := w.Write(buf.str[strpos:head.offset]) + strpos += n + if err != nil { + return err + } + } + // write the header + enc := head.encode(buf.sizebuf[:]) + if _, err = w.Write(enc); err != nil { + return err + } + } + if strpos < len(buf.str) { + // write string data after the last list header + _, err = w.Write(buf.str[strpos:]) + } + return err +} + +// Write implements io.Writer and appends b directly to the output. +func (buf *encBuffer) Write(b []byte) (int, error) { + buf.str = append(buf.str, b...) + return len(b), nil +} + +// writeBool writes b as the integer 0 (false) or 1 (true). +func (buf *encBuffer) writeBool(b bool) { + if b { + buf.str = append(buf.str, 0x01) + } else { + buf.str = append(buf.str, 0x80) + } +} + +func (buf *encBuffer) writeUint64(i uint64) { + if i == 0 { + buf.str = append(buf.str, 0x80) + } else if i < 128 { + // fits single byte + buf.str = append(buf.str, byte(i)) + } else { + s := putint(buf.sizebuf[1:], i) + buf.sizebuf[0] = 0x80 + byte(s) + buf.str = append(buf.str, buf.sizebuf[:s+1]...) + } +} + +func (buf *encBuffer) writeBytes(b []byte) { + if len(b) == 1 && b[0] <= 0x7F { + // fits single byte, no string header + buf.str = append(buf.str, b[0]) + } else { + buf.encodeStringHeader(len(b)) + buf.str = append(buf.str, b...) + } +} + +func (buf *encBuffer) writeString(s string) { + buf.writeBytes([]byte(s)) +} + +// wordBytes is the number of bytes in a big.Word +const wordBytes = (32 << (uint64(^big.Word(0)) >> 63)) / 8 + +// writeBigInt writes i as an integer. +func (w *encBuffer) writeBigInt(i *big.Int) { + bitlen := i.BitLen() + if bitlen <= 64 { + w.writeUint64(i.Uint64()) + return + } + // Integer is larger than 64 bits, encode from i.Bits(). + // The minimal byte length is bitlen rounded up to the next + // multiple of 8, divided by 8. + length := ((bitlen + 7) & -8) >> 3 + w.encodeStringHeader(length) + w.str = append(w.str, make([]byte, length)...) + index := length + buf := w.str[len(w.str)-length:] + for _, d := range i.Bits() { + for j := 0; j < wordBytes && index > 0; j++ { + index-- + buf[index] = byte(d) + d >>= 8 + } + } +} + +// list adds a new list header to the header stack. It returns the index of the header. +// Call listEnd with this index after encoding the content of the list. +func (buf *encBuffer) list() int { + buf.lheads = append(buf.lheads, listhead{offset: len(buf.str), size: buf.lhsize}) + return len(buf.lheads) - 1 +} + +func (buf *encBuffer) listEnd(index int) { + lh := &buf.lheads[index] + lh.size = buf.size() - lh.offset - lh.size + if lh.size < 56 { + buf.lhsize++ // length encoded into kind tag + } else { + buf.lhsize += 1 + intsize(uint64(lh.size)) + } +} + +func (buf *encBuffer) encode(val interface{}) error { + rval := reflect.ValueOf(val) + writer, err := cachedWriter(rval.Type()) + if err != nil { + return err + } + return writer(rval, buf) +} + +func (buf *encBuffer) encodeStringHeader(size int) { + if size < 56 { + buf.str = append(buf.str, 0x80+byte(size)) + } else { + sizesize := putint(buf.sizebuf[1:], uint64(size)) + buf.sizebuf[0] = 0xB7 + byte(sizesize) + buf.str = append(buf.str, buf.sizebuf[:sizesize+1]...) + } +} + +// encReader is the io.Reader returned by EncodeToReader. +// It releases its encbuf at EOF. +type encReader struct { + buf *encBuffer // the buffer we're reading from. this is nil when we're at EOF. + lhpos int // index of list header that we're reading + strpos int // current position in string buffer + piece []byte // next piece to be read +} + +func (r *encReader) Read(b []byte) (n int, err error) { + for { + if r.piece = r.next(); r.piece == nil { + // Put the encode buffer back into the pool at EOF when it + // is first encountered. Subsequent calls still return EOF + // as the error but the buffer is no longer valid. + if r.buf != nil { + encBufferPool.Put(r.buf) + r.buf = nil + } + return n, io.EOF + } + nn := copy(b[n:], r.piece) + n += nn + if nn < len(r.piece) { + // piece didn't fit, see you next time. + r.piece = r.piece[nn:] + return n, nil + } + r.piece = nil + } +} + +// next returns the next piece of data to be read. +// it returns nil at EOF. +func (r *encReader) next() []byte { + switch { + case r.buf == nil: + return nil + + case r.piece != nil: + // There is still data available for reading. + return r.piece + + case r.lhpos < len(r.buf.lheads): + // We're before the last list header. + head := r.buf.lheads[r.lhpos] + sizebefore := head.offset - r.strpos + if sizebefore > 0 { + // String data before header. + p := r.buf.str[r.strpos:head.offset] + r.strpos += sizebefore + return p + } + r.lhpos++ + return head.encode(r.buf.sizebuf[:]) + + case r.strpos < len(r.buf.str): + // String data at the end, after all list headers. + p := r.buf.str[r.strpos:] + r.strpos = len(r.buf.str) + return p + + default: + return nil + } +} + +func encBufferFromWriter(w io.Writer) *encBuffer { + switch w := w.(type) { + case EncoderBuffer: + return w.buf + case *EncoderBuffer: + return w.buf + case *encBuffer: + return w + default: + return nil + } +} + +// EncoderBuffer is a buffer for incremental encoding. +// +// The zero value is NOT ready for use. To get a usable buffer, +// create it using NewEncoderBuffer or call Reset. +type EncoderBuffer struct { + buf *encBuffer + dst io.Writer + + ownBuffer bool +} + +// NewEncoderBuffer creates an encoder buffer. +func NewEncoderBuffer(dst io.Writer) EncoderBuffer { + var w EncoderBuffer + w.Reset(dst) + return w +} + +// Reset truncates the buffer and sets the output destination. +func (w *EncoderBuffer) Reset(dst io.Writer) { + if w.buf != nil && !w.ownBuffer { + panic("can't Reset derived EncoderBuffer") + } + + // If the destination writer has an *encBuffer, use it. + // Note that w.ownBuffer is left false here. + if dst != nil { + if outer := encBufferFromWriter(dst); outer != nil { + *w = EncoderBuffer{outer, nil, false} + return + } + } + + // Get a fresh buffer. + if w.buf == nil { + w.buf = encBufferPool.Get().(*encBuffer) + w.ownBuffer = true + } + w.buf.reset() + w.dst = dst +} + +// Flush writes encoded RLP data to the output writer. This can only be called once. +// If you want to re-use the buffer after Flush, you must call Reset. +func (w *EncoderBuffer) Flush() error { + var err error + if w.dst != nil { + err = w.buf.writeTo(w.dst) + } + // Release the internal buffer. + if w.ownBuffer { + encBufferPool.Put(w.buf) + } + *w = EncoderBuffer{} + return err +} + +// ToBytes returns the encoded bytes. +func (w *EncoderBuffer) ToBytes() []byte { + return w.buf.makeBytes() +} + +// AppendToBytes appends the encoded bytes to dst. +func (w *EncoderBuffer) AppendToBytes(dst []byte) []byte { + size := w.buf.size() + out := append(dst, make([]byte, size)...) + w.buf.copyTo(out[len(dst):]) + return out +} + +// Write appends b directly to the encoder output. +func (w EncoderBuffer) Write(b []byte) (int, error) { + return w.buf.Write(b) +} + +// WriteBool writes b as the integer 0 (false) or 1 (true). +func (w EncoderBuffer) WriteBool(b bool) { + w.buf.writeBool(b) +} + +// WriteUint64 encodes an unsigned integer. +func (w EncoderBuffer) WriteUint64(i uint64) { + w.buf.writeUint64(i) +} + +// WriteBigInt encodes a big.Int as an RLP string. +// Note: Unlike with Encode, the sign of i is ignored. +func (w EncoderBuffer) WriteBigInt(i *big.Int) { + w.buf.writeBigInt(i) +} + +// WriteBytes encodes b as an RLP string. +func (w EncoderBuffer) WriteBytes(b []byte) { + w.buf.writeBytes(b) +} + +// WriteBytes encodes s as an RLP string. +func (w EncoderBuffer) WriteString(s string) { + w.buf.writeString(s) +} + +// List starts a list. It returns an internal index. Call EndList with +// this index after encoding the content to finish the list. +func (w EncoderBuffer) List() int { + return w.buf.list() +} + +// ListEnd finishes the given list. +func (w EncoderBuffer) ListEnd(index int) { + w.buf.listEnd(index) +} diff --git a/encoding/rlp/encbuffer_example_test.go b/encoding/rlp/encbuffer_example_test.go new file mode 100644 index 000000000..014fd6d2a --- /dev/null +++ b/encoding/rlp/encbuffer_example_test.go @@ -0,0 +1,45 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rlp_test + +import ( + "bytes" + "fmt" + + "github.com/onflow/flow-go-sdk/encoding/rlp" +) + +func ExampleEncoderBuffer() { + var w bytes.Buffer + + // Encode [4, [5, 6]] to w. + buf := rlp.NewEncoderBuffer(&w) + l1 := buf.List() + buf.WriteUint64(4) + l2 := buf.List() + buf.WriteUint64(5) + buf.WriteUint64(6) + buf.ListEnd(l2) + buf.ListEnd(l1) + + if err := buf.Flush(); err != nil { + panic(err) + } + fmt.Printf("%X\n", w.Bytes()) + // Output: + // C404C20506 +} diff --git a/encoding/rlp/encode.go b/encoding/rlp/encode.go new file mode 100644 index 000000000..642ed28d3 --- /dev/null +++ b/encoding/rlp/encode.go @@ -0,0 +1,471 @@ +// Copyright 2014 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rlp + +import ( + "errors" + "fmt" + "io" + "math/big" + "reflect" + + "github.com/onflow/flow-go-sdk/encoding/rlp/internal/rlpstruct" +) + +var ( + // Common encoded values. + // These are useful when implementing EncodeRLP. + EmptyString = []byte{0x80} + EmptyList = []byte{0xC0} +) + +var ErrNegativeBigInt = errors.New("rlp: cannot encode negative big.Int") + +// Encoder is implemented by types that require custom +// encoding rules or want to encode private fields. +type Encoder interface { + // EncodeRLP should write the RLP encoding of its receiver to w. + // If the implementation is a pointer method, it may also be + // called for nil pointers. + // + // Implementations should generate valid RLP. The data written is + // not verified at the moment, but a future version might. It is + // recommended to write only a single value but writing multiple + // values or no value at all is also permitted. + EncodeRLP(io.Writer) error +} + +// Encode writes the RLP encoding of val to w. Note that Encode may +// perform many small writes in some cases. Consider making w +// buffered. +// +// Please see package-level documentation of encoding rules. +func Encode(w io.Writer, val interface{}) error { + // Optimization: reuse *encBuffer when called by EncodeRLP. + if buf := encBufferFromWriter(w); buf != nil { + return buf.encode(val) + } + + buf := getEncBuffer() + defer encBufferPool.Put(buf) + if err := buf.encode(val); err != nil { + return err + } + return buf.writeTo(w) +} + +// EncodeToBytes returns the RLP encoding of val. +// Please see package-level documentation for the encoding rules. +func EncodeToBytes(val interface{}) ([]byte, error) { + buf := getEncBuffer() + defer encBufferPool.Put(buf) + + if err := buf.encode(val); err != nil { + return nil, err + } + return buf.makeBytes(), nil +} + +// EncodeToReader returns a reader from which the RLP encoding of val +// can be read. The returned size is the total size of the encoded +// data. +// +// Please see the documentation of Encode for the encoding rules. +func EncodeToReader(val interface{}) (size int, r io.Reader, err error) { + buf := getEncBuffer() + if err := buf.encode(val); err != nil { + encBufferPool.Put(buf) + return 0, nil, err + } + // Note: can't put the reader back into the pool here + // because it is held by encReader. The reader puts it + // back when it has been fully consumed. + return buf.size(), &encReader{buf: buf}, nil +} + +type listhead struct { + offset int // index of this header in string data + size int // total size of encoded data (including list headers) +} + +// encode writes head to the given buffer, which must be at least +// 9 bytes long. It returns the encoded bytes. +func (head *listhead) encode(buf []byte) []byte { + return buf[:puthead(buf, 0xC0, 0xF7, uint64(head.size))] +} + +// headsize returns the size of a list or string header +// for a value of the given size. +func headsize(size uint64) int { + if size < 56 { + return 1 + } + return 1 + intsize(size) +} + +// puthead writes a list or string header to buf. +// buf must be at least 9 bytes long. +func puthead(buf []byte, smalltag, largetag byte, size uint64) int { + if size < 56 { + buf[0] = smalltag + byte(size) + return 1 + } + sizesize := putint(buf[1:], size) + buf[0] = largetag + byte(sizesize) + return sizesize + 1 +} + +var encoderInterface = reflect.TypeOf(new(Encoder)).Elem() + +// makeWriter creates a writer function for the given type. +func makeWriter(typ reflect.Type, ts rlpstruct.Tags) (writer, error) { + kind := typ.Kind() + switch { + case typ == rawValueType: + return writeRawValue, nil + case typ.AssignableTo(reflect.PtrTo(bigInt)): + return writeBigIntPtr, nil + case typ.AssignableTo(bigInt): + return writeBigIntNoPtr, nil + case kind == reflect.Ptr: + return makePtrWriter(typ, ts) + case reflect.PtrTo(typ).Implements(encoderInterface): + return makeEncoderWriter(typ), nil + case isUint(kind): + return writeUint, nil + case kind == reflect.Bool: + return writeBool, nil + case kind == reflect.String: + return writeString, nil + case kind == reflect.Slice && isByte(typ.Elem()): + return writeBytes, nil + case kind == reflect.Array && isByte(typ.Elem()): + return makeByteArrayWriter(typ), nil + case kind == reflect.Slice || kind == reflect.Array: + return makeSliceWriter(typ, ts) + case kind == reflect.Struct: + return makeStructWriter(typ) + case kind == reflect.Interface: + return writeInterface, nil + default: + return nil, fmt.Errorf("rlp: type %v is not RLP-serializable", typ) + } +} + +func writeRawValue(val reflect.Value, w *encBuffer) error { + w.str = append(w.str, val.Bytes()...) + return nil +} + +func writeUint(val reflect.Value, w *encBuffer) error { + w.writeUint64(val.Uint()) + return nil +} + +func writeBool(val reflect.Value, w *encBuffer) error { + w.writeBool(val.Bool()) + return nil +} + +func writeBigIntPtr(val reflect.Value, w *encBuffer) error { + ptr := val.Interface().(*big.Int) + if ptr == nil { + w.str = append(w.str, 0x80) + return nil + } + if ptr.Sign() == -1 { + return ErrNegativeBigInt + } + w.writeBigInt(ptr) + return nil +} + +func writeBigIntNoPtr(val reflect.Value, w *encBuffer) error { + i := val.Interface().(big.Int) + if i.Sign() == -1 { + return ErrNegativeBigInt + } + w.writeBigInt(&i) + return nil +} + +func writeBytes(val reflect.Value, w *encBuffer) error { + w.writeBytes(val.Bytes()) + return nil +} + +func makeByteArrayWriter(typ reflect.Type) writer { + switch typ.Len() { + case 0: + return writeLengthZeroByteArray + case 1: + return writeLengthOneByteArray + default: + length := typ.Len() + return func(val reflect.Value, w *encBuffer) error { + if !val.CanAddr() { + // Getting the byte slice of val requires it to be addressable. Make it + // addressable by copying. + copy := reflect.New(val.Type()).Elem() + copy.Set(val) + val = copy + } + slice := byteArrayBytes(val, length) + w.encodeStringHeader(len(slice)) + w.str = append(w.str, slice...) + return nil + } + } +} + +func writeLengthZeroByteArray(val reflect.Value, w *encBuffer) error { + w.str = append(w.str, 0x80) + return nil +} + +func writeLengthOneByteArray(val reflect.Value, w *encBuffer) error { + b := byte(val.Index(0).Uint()) + if b <= 0x7f { + w.str = append(w.str, b) + } else { + w.str = append(w.str, 0x81, b) + } + return nil +} + +func writeString(val reflect.Value, w *encBuffer) error { + s := val.String() + if len(s) == 1 && s[0] <= 0x7f { + // fits single byte, no string header + w.str = append(w.str, s[0]) + } else { + w.encodeStringHeader(len(s)) + w.str = append(w.str, s...) + } + return nil +} + +func writeInterface(val reflect.Value, w *encBuffer) error { + if val.IsNil() { + // Write empty list. This is consistent with the previous RLP + // encoder that we had and should therefore avoid any + // problems. + w.str = append(w.str, 0xC0) + return nil + } + eval := val.Elem() + writer, err := cachedWriter(eval.Type()) + if err != nil { + return err + } + return writer(eval, w) +} + +func makeSliceWriter(typ reflect.Type, ts rlpstruct.Tags) (writer, error) { + etypeinfo := theTC.infoWhileGenerating(typ.Elem(), rlpstruct.Tags{}) + if etypeinfo.writerErr != nil { + return nil, etypeinfo.writerErr + } + + var wfn writer + if ts.Tail { + // This is for struct tail slices. + // w.list is not called for them. + wfn = func(val reflect.Value, w *encBuffer) error { + vlen := val.Len() + for i := 0; i < vlen; i++ { + if err := etypeinfo.writer(val.Index(i), w); err != nil { + return err + } + } + return nil + } + } else { + // This is for regular slices and arrays. + wfn = func(val reflect.Value, w *encBuffer) error { + vlen := val.Len() + if vlen == 0 { + w.str = append(w.str, 0xC0) + return nil + } + listOffset := w.list() + for i := 0; i < vlen; i++ { + if err := etypeinfo.writer(val.Index(i), w); err != nil { + return err + } + } + w.listEnd(listOffset) + return nil + } + } + return wfn, nil +} + +func makeStructWriter(typ reflect.Type) (writer, error) { + fields, err := structFields(typ) + if err != nil { + return nil, err + } + for _, f := range fields { + if f.info.writerErr != nil { + return nil, structFieldError{typ, f.index, f.info.writerErr} + } + } + + var writer writer + firstOptionalField := firstOptionalField(fields) + if firstOptionalField == len(fields) { + // This is the writer function for structs without any optional fields. + writer = func(val reflect.Value, w *encBuffer) error { + lh := w.list() + for _, f := range fields { + if err := f.info.writer(val.Field(f.index), w); err != nil { + return err + } + } + w.listEnd(lh) + return nil + } + } else { + // If there are any "optional" fields, the writer needs to perform additional + // checks to determine the output list length. + writer = func(val reflect.Value, w *encBuffer) error { + lastField := len(fields) - 1 + for ; lastField >= firstOptionalField; lastField-- { + if !val.Field(fields[lastField].index).IsZero() { + break + } + } + lh := w.list() + for i := 0; i <= lastField; i++ { + if err := fields[i].info.writer(val.Field(fields[i].index), w); err != nil { + return err + } + } + w.listEnd(lh) + return nil + } + } + return writer, nil +} + +func makePtrWriter(typ reflect.Type, ts rlpstruct.Tags) (writer, error) { + nilEncoding := byte(0xC0) + if typeNilKind(typ.Elem(), ts) == String { + nilEncoding = 0x80 + } + + etypeinfo := theTC.infoWhileGenerating(typ.Elem(), rlpstruct.Tags{}) + if etypeinfo.writerErr != nil { + return nil, etypeinfo.writerErr + } + + writer := func(val reflect.Value, w *encBuffer) error { + if ev := val.Elem(); ev.IsValid() { + return etypeinfo.writer(ev, w) + } + w.str = append(w.str, nilEncoding) + return nil + } + return writer, nil +} + +func makeEncoderWriter(typ reflect.Type) writer { + if typ.Implements(encoderInterface) { + return func(val reflect.Value, w *encBuffer) error { + return val.Interface().(Encoder).EncodeRLP(w) + } + } + w := func(val reflect.Value, w *encBuffer) error { + if !val.CanAddr() { + // package json simply doesn't call MarshalJSON for this case, but encodes the + // value as if it didn't implement the interface. We don't want to handle it that + // way. + return fmt.Errorf("rlp: unadressable value of type %v, EncodeRLP is pointer method", val.Type()) + } + return val.Addr().Interface().(Encoder).EncodeRLP(w) + } + return w +} + +// putint writes i to the beginning of b in big endian byte +// order, using the least number of bytes needed to represent i. +func putint(b []byte, i uint64) (size int) { + switch { + case i < (1 << 8): + b[0] = byte(i) + return 1 + case i < (1 << 16): + b[0] = byte(i >> 8) + b[1] = byte(i) + return 2 + case i < (1 << 24): + b[0] = byte(i >> 16) + b[1] = byte(i >> 8) + b[2] = byte(i) + return 3 + case i < (1 << 32): + b[0] = byte(i >> 24) + b[1] = byte(i >> 16) + b[2] = byte(i >> 8) + b[3] = byte(i) + return 4 + case i < (1 << 40): + b[0] = byte(i >> 32) + b[1] = byte(i >> 24) + b[2] = byte(i >> 16) + b[3] = byte(i >> 8) + b[4] = byte(i) + return 5 + case i < (1 << 48): + b[0] = byte(i >> 40) + b[1] = byte(i >> 32) + b[2] = byte(i >> 24) + b[3] = byte(i >> 16) + b[4] = byte(i >> 8) + b[5] = byte(i) + return 6 + case i < (1 << 56): + b[0] = byte(i >> 48) + b[1] = byte(i >> 40) + b[2] = byte(i >> 32) + b[3] = byte(i >> 24) + b[4] = byte(i >> 16) + b[5] = byte(i >> 8) + b[6] = byte(i) + return 7 + default: + b[0] = byte(i >> 56) + b[1] = byte(i >> 48) + b[2] = byte(i >> 40) + b[3] = byte(i >> 32) + b[4] = byte(i >> 24) + b[5] = byte(i >> 16) + b[6] = byte(i >> 8) + b[7] = byte(i) + return 8 + } +} + +// intsize computes the minimum number of bytes required to store i. +func intsize(i uint64) (size int) { + for size = 1; ; size++ { + if i >>= 8; i == 0 { + return size + } + } +} diff --git a/encoding/rlp/encode_test.go b/encoding/rlp/encode_test.go new file mode 100644 index 000000000..6ad29612a --- /dev/null +++ b/encoding/rlp/encode_test.go @@ -0,0 +1,585 @@ +// Copyright 2014 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rlp + +import ( + "bytes" + "errors" + "fmt" + "io" + "math/big" + "runtime" + "sync" + "testing" + + "github.com/onflow/flow-go-sdk/encoding/common/math" +) + +type testEncoder struct { + err error +} + +func (e *testEncoder) EncodeRLP(w io.Writer) error { + if e == nil { + panic("EncodeRLP called on nil value") + } + if e.err != nil { + return e.err + } + w.Write([]byte{0, 1, 0, 1, 0, 1, 0, 1, 0, 1}) + return nil +} + +type testEncoderValueMethod struct{} + +func (e testEncoderValueMethod) EncodeRLP(w io.Writer) error { + w.Write([]byte{0xFA, 0xFE, 0xF0}) + return nil +} + +type byteEncoder byte + +func (e byteEncoder) EncodeRLP(w io.Writer) error { + w.Write(EmptyList) + return nil +} + +type undecodableEncoder func() + +func (f undecodableEncoder) EncodeRLP(w io.Writer) error { + w.Write([]byte{0xF5, 0xF5, 0xF5}) + return nil +} + +type encodableReader struct { + A, B uint +} + +func (e *encodableReader) Read(b []byte) (int, error) { + panic("called") +} + +type namedByteType byte + +var ( + _ = Encoder(&testEncoder{}) + _ = Encoder(byteEncoder(0)) + + reader io.Reader = &encodableReader{1, 2} +) + +type encTest struct { + val interface{} + output, error string +} + +var encTests = []encTest{ + // booleans + {val: true, output: "01"}, + {val: false, output: "80"}, + + // integers + {val: uint32(0), output: "80"}, + {val: uint32(127), output: "7F"}, + {val: uint32(128), output: "8180"}, + {val: uint32(256), output: "820100"}, + {val: uint32(1024), output: "820400"}, + {val: uint32(0xFFFFFF), output: "83FFFFFF"}, + {val: uint32(0xFFFFFFFF), output: "84FFFFFFFF"}, + {val: uint64(0xFFFFFFFF), output: "84FFFFFFFF"}, + {val: uint64(0xFFFFFFFFFF), output: "85FFFFFFFFFF"}, + {val: uint64(0xFFFFFFFFFFFF), output: "86FFFFFFFFFFFF"}, + {val: uint64(0xFFFFFFFFFFFFFF), output: "87FFFFFFFFFFFFFF"}, + {val: uint64(0xFFFFFFFFFFFFFFFF), output: "88FFFFFFFFFFFFFFFF"}, + + // big integers (should match uint for small values) + {val: big.NewInt(0), output: "80"}, + {val: big.NewInt(1), output: "01"}, + {val: big.NewInt(127), output: "7F"}, + {val: big.NewInt(128), output: "8180"}, + {val: big.NewInt(256), output: "820100"}, + {val: big.NewInt(1024), output: "820400"}, + {val: big.NewInt(0xFFFFFF), output: "83FFFFFF"}, + {val: big.NewInt(0xFFFFFFFF), output: "84FFFFFFFF"}, + {val: big.NewInt(0xFFFFFFFFFF), output: "85FFFFFFFFFF"}, + {val: big.NewInt(0xFFFFFFFFFFFF), output: "86FFFFFFFFFFFF"}, + {val: big.NewInt(0xFFFFFFFFFFFFFF), output: "87FFFFFFFFFFFFFF"}, + { + val: big.NewInt(0).SetBytes(unhex("102030405060708090A0B0C0D0E0F2")), + output: "8F102030405060708090A0B0C0D0E0F2", + }, + { + val: big.NewInt(0).SetBytes(unhex("0100020003000400050006000700080009000A000B000C000D000E01")), + output: "9C0100020003000400050006000700080009000A000B000C000D000E01", + }, + { + val: big.NewInt(0).SetBytes(unhex("010000000000000000000000000000000000000000000000000000000000000000")), + output: "A1010000000000000000000000000000000000000000000000000000000000000000", + }, + { + val: veryBigInt, + output: "89FFFFFFFFFFFFFFFFFF", + }, + { + val: veryVeryBigInt, + output: "B848FFFFFFFFFFFFFFFFF800000000000000001BFFFFFFFFFFFFFFFFC8000000000000000045FFFFFFFFFFFFFFFFC800000000000000001BFFFFFFFFFFFFFFFFF8000000000000000001", + }, + + // non-pointer big.Int + {val: *big.NewInt(0), output: "80"}, + {val: *big.NewInt(0xFFFFFF), output: "83FFFFFF"}, + + // negative ints are not supported + {val: big.NewInt(-1), error: "rlp: cannot encode negative big.Int"}, + {val: *big.NewInt(-1), error: "rlp: cannot encode negative big.Int"}, + + // byte arrays + {val: [0]byte{}, output: "80"}, + {val: [1]byte{0}, output: "00"}, + {val: [1]byte{1}, output: "01"}, + {val: [1]byte{0x7F}, output: "7F"}, + {val: [1]byte{0x80}, output: "8180"}, + {val: [1]byte{0xFF}, output: "81FF"}, + {val: [3]byte{1, 2, 3}, output: "83010203"}, + {val: [57]byte{1, 2, 3}, output: "B839010203000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"}, + + // named byte type arrays + {val: [0]namedByteType{}, output: "80"}, + {val: [1]namedByteType{0}, output: "00"}, + {val: [1]namedByteType{1}, output: "01"}, + {val: [1]namedByteType{0x7F}, output: "7F"}, + {val: [1]namedByteType{0x80}, output: "8180"}, + {val: [1]namedByteType{0xFF}, output: "81FF"}, + {val: [3]namedByteType{1, 2, 3}, output: "83010203"}, + {val: [57]namedByteType{1, 2, 3}, output: "B839010203000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"}, + + // byte slices + {val: []byte{}, output: "80"}, + {val: []byte{0}, output: "00"}, + {val: []byte{0x7E}, output: "7E"}, + {val: []byte{0x7F}, output: "7F"}, + {val: []byte{0x80}, output: "8180"}, + {val: []byte{1, 2, 3}, output: "83010203"}, + + // named byte type slices + {val: []namedByteType{}, output: "80"}, + {val: []namedByteType{0}, output: "00"}, + {val: []namedByteType{0x7E}, output: "7E"}, + {val: []namedByteType{0x7F}, output: "7F"}, + {val: []namedByteType{0x80}, output: "8180"}, + {val: []namedByteType{1, 2, 3}, output: "83010203"}, + + // strings + {val: "", output: "80"}, + {val: "\x7E", output: "7E"}, + {val: "\x7F", output: "7F"}, + {val: "\x80", output: "8180"}, + {val: "dog", output: "83646F67"}, + { + val: "Lorem ipsum dolor sit amet, consectetur adipisicing eli", + output: "B74C6F72656D20697073756D20646F6C6F722073697420616D65742C20636F6E7365637465747572206164697069736963696E6720656C69", + }, + { + val: "Lorem ipsum dolor sit amet, consectetur adipisicing elit", + output: "B8384C6F72656D20697073756D20646F6C6F722073697420616D65742C20636F6E7365637465747572206164697069736963696E6720656C6974", + }, + { + val: "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Curabitur mauris magna, suscipit sed vehicula non, iaculis faucibus tortor. Proin suscipit ultricies malesuada. Duis tortor elit, dictum quis tristique eu, ultrices at risus. Morbi a est imperdiet mi ullamcorper aliquet suscipit nec lorem. Aenean quis leo mollis, vulputate elit varius, consequat enim. Nulla ultrices turpis justo, et posuere urna consectetur nec. Proin non convallis metus. Donec tempor ipsum in mauris congue sollicitudin. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Suspendisse convallis sem vel massa faucibus, eget lacinia lacus tempor. Nulla quis ultricies purus. Proin auctor rhoncus nibh condimentum mollis. Aliquam consequat enim at metus luctus, a eleifend purus egestas. Curabitur at nibh metus. Nam bibendum, neque at auctor tristique, lorem libero aliquet arcu, non interdum tellus lectus sit amet eros. Cras rhoncus, metus ac ornare cursus, dolor justo ultrices metus, at ullamcorper volutpat", + output: "B904004C6F72656D20697073756D20646F6C6F722073697420616D65742C20636F6E73656374657475722061646970697363696E6720656C69742E20437572616269747572206D6175726973206D61676E612C20737573636970697420736564207665686963756C61206E6F6E2C20696163756C697320666175636962757320746F72746F722E2050726F696E20737573636970697420756C74726963696573206D616C6573756164612E204475697320746F72746F7220656C69742C2064696374756D2071756973207472697374697175652065752C20756C7472696365732061742072697375732E204D6F72626920612065737420696D70657264696574206D6920756C6C616D636F7270657220616C6971756574207375736369706974206E6563206C6F72656D2E2041656E65616E2071756973206C656F206D6F6C6C69732C2076756C70757461746520656C6974207661726975732C20636F6E73657175617420656E696D2E204E756C6C6120756C74726963657320747572706973206A7573746F2C20657420706F73756572652075726E6120636F6E7365637465747572206E65632E2050726F696E206E6F6E20636F6E76616C6C6973206D657475732E20446F6E65632074656D706F7220697073756D20696E206D617572697320636F6E67756520736F6C6C696369747564696E2E20566573746962756C756D20616E746520697073756D207072696D697320696E206661756369627573206F726369206C756374757320657420756C74726963657320706F737565726520637562696C69612043757261653B2053757370656E646973736520636F6E76616C6C69732073656D2076656C206D617373612066617563696275732C2065676574206C6163696E6961206C616375732074656D706F722E204E756C6C61207175697320756C747269636965732070757275732E2050726F696E20617563746F722072686F6E637573206E69626820636F6E64696D656E74756D206D6F6C6C69732E20416C697175616D20636F6E73657175617420656E696D206174206D65747573206C75637475732C206120656C656966656E6420707572757320656765737461732E20437572616269747572206174206E696268206D657475732E204E616D20626962656E64756D2C206E6571756520617420617563746F72207472697374697175652C206C6F72656D206C696265726F20616C697175657420617263752C206E6F6E20696E74657264756D2074656C6C7573206C65637475732073697420616D65742065726F732E20437261732072686F6E6375732C206D65747573206163206F726E617265206375727375732C20646F6C6F72206A7573746F20756C747269636573206D657475732C20617420756C6C616D636F7270657220766F6C7574706174", + }, + + // slices + {val: []uint{}, output: "C0"}, + {val: []uint{1, 2, 3}, output: "C3010203"}, + { + // [ [], [[]], [ [], [[]] ] ] + val: []interface{}{[]interface{}{}, [][]interface{}{{}}, []interface{}{[]interface{}{}, [][]interface{}{{}}}}, + output: "C7C0C1C0C3C0C1C0", + }, + { + val: []string{"aaa", "bbb", "ccc", "ddd", "eee", "fff", "ggg", "hhh", "iii", "jjj", "kkk", "lll", "mmm", "nnn", "ooo"}, + output: "F83C836161618362626283636363836464648365656583666666836767678368686883696969836A6A6A836B6B6B836C6C6C836D6D6D836E6E6E836F6F6F", + }, + { + val: []interface{}{uint(1), uint(0xFFFFFF), []interface{}{[]uint{4, 5, 5}}, "abc"}, + output: "CE0183FFFFFFC4C304050583616263", + }, + { + val: [][]string{ + {"asdf", "qwer", "zxcv"}, + {"asdf", "qwer", "zxcv"}, + {"asdf", "qwer", "zxcv"}, + {"asdf", "qwer", "zxcv"}, + {"asdf", "qwer", "zxcv"}, + {"asdf", "qwer", "zxcv"}, + {"asdf", "qwer", "zxcv"}, + {"asdf", "qwer", "zxcv"}, + {"asdf", "qwer", "zxcv"}, + {"asdf", "qwer", "zxcv"}, + {"asdf", "qwer", "zxcv"}, + {"asdf", "qwer", "zxcv"}, + {"asdf", "qwer", "zxcv"}, + {"asdf", "qwer", "zxcv"}, + {"asdf", "qwer", "zxcv"}, + {"asdf", "qwer", "zxcv"}, + {"asdf", "qwer", "zxcv"}, + {"asdf", "qwer", "zxcv"}, + {"asdf", "qwer", "zxcv"}, + {"asdf", "qwer", "zxcv"}, + {"asdf", "qwer", "zxcv"}, + {"asdf", "qwer", "zxcv"}, + {"asdf", "qwer", "zxcv"}, + {"asdf", "qwer", "zxcv"}, + {"asdf", "qwer", "zxcv"}, + {"asdf", "qwer", "zxcv"}, + {"asdf", "qwer", "zxcv"}, + {"asdf", "qwer", "zxcv"}, + {"asdf", "qwer", "zxcv"}, + {"asdf", "qwer", "zxcv"}, + {"asdf", "qwer", "zxcv"}, + {"asdf", "qwer", "zxcv"}, + }, + output: "F90200CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376", + }, + + // RawValue + {val: RawValue(unhex("01")), output: "01"}, + {val: RawValue(unhex("82FFFF")), output: "82FFFF"}, + {val: []RawValue{unhex("01"), unhex("02")}, output: "C20102"}, + + // structs + {val: simplestruct{}, output: "C28080"}, + {val: simplestruct{A: 3, B: "foo"}, output: "C50383666F6F"}, + {val: &recstruct{5, nil}, output: "C205C0"}, + {val: &recstruct{5, &recstruct{4, &recstruct{3, nil}}}, output: "C605C404C203C0"}, + {val: &intField{X: 3}, error: "rlp: type int is not RLP-serializable (struct field rlp.intField.X)"}, + + // struct tag "-" + {val: &ignoredField{A: 1, B: 2, C: 3}, output: "C20103"}, + + // struct tag "tail" + {val: &tailRaw{A: 1, Tail: []RawValue{unhex("02"), unhex("03")}}, output: "C3010203"}, + {val: &tailRaw{A: 1, Tail: []RawValue{unhex("02")}}, output: "C20102"}, + {val: &tailRaw{A: 1, Tail: []RawValue{}}, output: "C101"}, + {val: &tailRaw{A: 1, Tail: nil}, output: "C101"}, + + // struct tag "optional" + {val: &optionalFields{}, output: "C180"}, + {val: &optionalFields{A: 1}, output: "C101"}, + {val: &optionalFields{A: 1, B: 2}, output: "C20102"}, + {val: &optionalFields{A: 1, B: 2, C: 3}, output: "C3010203"}, + {val: &optionalFields{A: 1, B: 0, C: 3}, output: "C3018003"}, + {val: &optionalAndTailField{A: 1}, output: "C101"}, + {val: &optionalAndTailField{A: 1, B: 2}, output: "C20102"}, + {val: &optionalAndTailField{A: 1, Tail: []uint{5, 6}}, output: "C401800506"}, + {val: &optionalAndTailField{A: 1, Tail: []uint{5, 6}}, output: "C401800506"}, + {val: &optionalBigIntField{A: 1}, output: "C101"}, + {val: &optionalPtrField{A: 1}, output: "C101"}, + {val: &optionalPtrFieldNil{A: 1}, output: "C101"}, + + // nil + {val: (*uint)(nil), output: "80"}, + {val: (*string)(nil), output: "80"}, + {val: (*[]byte)(nil), output: "80"}, + {val: (*[10]byte)(nil), output: "80"}, + {val: (*big.Int)(nil), output: "80"}, + {val: (*[]string)(nil), output: "C0"}, + {val: (*[10]string)(nil), output: "C0"}, + {val: (*[]interface{})(nil), output: "C0"}, + {val: (*[]struct{ uint })(nil), output: "C0"}, + {val: (*interface{})(nil), output: "C0"}, + + // nil struct fields + { + val: struct { + X *[]byte + }{}, + output: "C180", + }, + { + val: struct { + X *[2]byte + }{}, + output: "C180", + }, + { + val: struct { + X *uint64 + }{}, + output: "C180", + }, + { + val: struct { + X *uint64 `rlp:"nilList"` + }{}, + output: "C1C0", + }, + { + val: struct { + X *[]uint64 + }{}, + output: "C1C0", + }, + { + val: struct { + X *[]uint64 `rlp:"nilString"` + }{}, + output: "C180", + }, + + // interfaces + {val: []io.Reader{reader}, output: "C3C20102"}, // the contained value is a struct + + // Encoder + {val: (*testEncoder)(nil), output: "C0"}, + {val: &testEncoder{}, output: "00010001000100010001"}, + {val: &testEncoder{errors.New("test error")}, error: "test error"}, + {val: struct{ E testEncoderValueMethod }{}, output: "C3FAFEF0"}, + {val: struct{ E *testEncoderValueMethod }{}, output: "C1C0"}, + + // Verify that the Encoder interface works for unsupported types like func(). + {val: undecodableEncoder(func() {}), output: "F5F5F5"}, + + // Verify that pointer method testEncoder.EncodeRLP is called for + // addressable non-pointer values. + {val: &struct{ TE testEncoder }{testEncoder{}}, output: "CA00010001000100010001"}, + {val: &struct{ TE testEncoder }{testEncoder{errors.New("test error")}}, error: "test error"}, + + // Verify the error for non-addressable non-pointer Encoder. + {val: testEncoder{}, error: "rlp: unadressable value of type rlp.testEncoder, EncodeRLP is pointer method"}, + + // Verify Encoder takes precedence over []byte. + {val: []byteEncoder{0, 1, 2, 3, 4}, output: "C5C0C0C0C0C0"}, +} + +func runEncTests(t *testing.T, f func(val interface{}) ([]byte, error)) { + for i, test := range encTests { + output, err := f(test.val) + if err != nil && test.error == "" { + t.Errorf("test %d: unexpected error: %v\nvalue %#v\ntype %T", + i, err, test.val, test.val) + continue + } + if test.error != "" && fmt.Sprint(err) != test.error { + t.Errorf("test %d: error mismatch\ngot %v\nwant %v\nvalue %#v\ntype %T", + i, err, test.error, test.val, test.val) + continue + } + if err == nil && !bytes.Equal(output, unhex(test.output)) { + t.Errorf("test %d: output mismatch:\ngot %X\nwant %s\nvalue %#v\ntype %T", + i, output, test.output, test.val, test.val) + } + } +} + +func TestEncode(t *testing.T) { + runEncTests(t, func(val interface{}) ([]byte, error) { + b := new(bytes.Buffer) + err := Encode(b, val) + return b.Bytes(), err + }) +} + +func TestEncodeToBytes(t *testing.T) { + runEncTests(t, EncodeToBytes) +} + +func TestEncodeAppendToBytes(t *testing.T) { + buffer := make([]byte, 20) + runEncTests(t, func(val interface{}) ([]byte, error) { + w := NewEncoderBuffer(nil) + defer w.Flush() + + err := Encode(w, val) + if err != nil { + return nil, err + } + output := w.AppendToBytes(buffer[:0]) + return output, nil + }) +} + +func TestEncodeToReader(t *testing.T) { + runEncTests(t, func(val interface{}) ([]byte, error) { + _, r, err := EncodeToReader(val) + if err != nil { + return nil, err + } + return io.ReadAll(r) + }) +} + +func TestEncodeToReaderPiecewise(t *testing.T) { + runEncTests(t, func(val interface{}) ([]byte, error) { + size, r, err := EncodeToReader(val) + if err != nil { + return nil, err + } + + // read output piecewise + output := make([]byte, size) + for start, end := 0, 0; start < size; start = end { + if remaining := size - start; remaining < 3 { + end += remaining + } else { + end = start + 3 + } + n, err := r.Read(output[start:end]) + end = start + n + if err == io.EOF { + break + } else if err != nil { + return nil, err + } + } + return output, nil + }) +} + +// This is a regression test verifying that encReader +// returns its encbuf to the pool only once. +func TestEncodeToReaderReturnToPool(t *testing.T) { + buf := make([]byte, 50) + wg := new(sync.WaitGroup) + for i := 0; i < 5; i++ { + wg.Add(1) + go func() { + for i := 0; i < 1000; i++ { + _, r, _ := EncodeToReader("foo") + io.ReadAll(r) + r.Read(buf) + r.Read(buf) + r.Read(buf) + r.Read(buf) + } + wg.Done() + }() + } + wg.Wait() +} + +var sink interface{} + +func BenchmarkIntsize(b *testing.B) { + for i := 0; i < b.N; i++ { + sink = intsize(0x12345678) + } +} + +func BenchmarkPutint(b *testing.B) { + buf := make([]byte, 8) + for i := 0; i < b.N; i++ { + putint(buf, 0x12345678) + sink = buf + } +} + +func BenchmarkEncodeBigInts(b *testing.B) { + ints := make([]*big.Int, 200) + for i := range ints { + ints[i] = math.BigPow(2, int64(i)) + } + out := bytes.NewBuffer(make([]byte, 0, 4096)) + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + out.Reset() + if err := Encode(out, ints); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkEncodeConcurrentInterface(b *testing.B) { + type struct1 struct { + A string + B *big.Int + C [20]byte + } + value := []interface{}{ + uint(999), + &struct1{A: "hello", B: big.NewInt(0xFFFFFFFF)}, + [10]byte{1, 2, 3, 4, 5, 6}, + []string{"yeah", "yeah", "yeah"}, + } + + var wg sync.WaitGroup + for cpu := 0; cpu < runtime.NumCPU(); cpu++ { + wg.Add(1) + go func() { + defer wg.Done() + + var buffer bytes.Buffer + for i := 0; i < b.N; i++ { + buffer.Reset() + err := Encode(&buffer, value) + if err != nil { + panic(err) + } + } + }() + } + wg.Wait() +} + +type byteArrayStruct struct { + A [20]byte + B [32]byte + C [32]byte +} + +func BenchmarkEncodeByteArrayStruct(b *testing.B) { + var out bytes.Buffer + var value byteArrayStruct + + b.ReportAllocs() + for i := 0; i < b.N; i++ { + out.Reset() + if err := Encode(&out, &value); err != nil { + b.Fatal(err) + } + } +} + +type structSliceElem struct { + X uint64 + Y uint64 + Z uint64 +} + +type structPtrSlice []*structSliceElem + +func BenchmarkEncodeStructPtrSlice(b *testing.B) { + var out bytes.Buffer + var value = structPtrSlice{ + &structSliceElem{1, 1, 1}, + &structSliceElem{2, 2, 2}, + &structSliceElem{3, 3, 3}, + &structSliceElem{5, 5, 5}, + &structSliceElem{6, 6, 6}, + &structSliceElem{7, 7, 7}, + } + + b.ReportAllocs() + for i := 0; i < b.N; i++ { + out.Reset() + if err := Encode(&out, &value); err != nil { + b.Fatal(err) + } + } +} diff --git a/encoding/rlp/encoder_example_test.go b/encoding/rlp/encoder_example_test.go new file mode 100644 index 000000000..d2fdbdbbf --- /dev/null +++ b/encoding/rlp/encoder_example_test.go @@ -0,0 +1,48 @@ +// Copyright 2014 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rlp_test + +import ( + "fmt" + "io" + + "github.com/onflow/flow-go-sdk/encoding/rlp" +) + +type MyCoolType struct { + Name string + a, b uint +} + +// EncodeRLP writes x as RLP list [a, b] that omits the Name field. +func (x *MyCoolType) EncodeRLP(w io.Writer) (err error) { + return rlp.Encode(w, []uint{x.a, x.b}) +} + +func ExampleEncoder() { + var t *MyCoolType // t is nil pointer to MyCoolType + bytes, _ := rlp.EncodeToBytes(t) + fmt.Printf("%v → %X\n", t, bytes) + + t = &MyCoolType{Name: "foobar", a: 5, b: 6} + bytes, _ = rlp.EncodeToBytes(t) + fmt.Printf("%v → %X\n", t, bytes) + + // Output: + // → C0 + // &{foobar 5 6} → C20506 +} diff --git a/encoding/rlp/internal/rlpstruct/rlpstruct.go b/encoding/rlp/internal/rlpstruct/rlpstruct.go new file mode 100644 index 000000000..1edead96c --- /dev/null +++ b/encoding/rlp/internal/rlpstruct/rlpstruct.go @@ -0,0 +1,213 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Package rlpstruct implements struct processing for RLP encoding/decoding. +// +// In particular, this package handles all rules around field filtering, +// struct tags and nil value determination. +package rlpstruct + +import ( + "fmt" + "reflect" + "strings" +) + +// Field represents a struct field. +type Field struct { + Name string + Index int + Exported bool + Type Type + Tag string +} + +// Type represents the attributes of a Go type. +type Type struct { + Name string + Kind reflect.Kind + IsEncoder bool // whether type implements rlp.Encoder + IsDecoder bool // whether type implements rlp.Decoder + Elem *Type // non-nil for Kind values of Ptr, Slice, Array +} + +// defaultNilValue determines whether a nil pointer to t encodes/decodes +// as an empty string or empty list. +func (t Type) DefaultNilValue() NilKind { + k := t.Kind + if isUint(k) || k == reflect.String || k == reflect.Bool || isByteArray(t) { + return NilKindString + } + return NilKindList +} + +// NilKind is the RLP value encoded in place of nil pointers. +type NilKind uint8 + +const ( + NilKindString NilKind = 0x80 + NilKindList NilKind = 0xC0 +) + +// Tags represents struct tags. +type Tags struct { + // rlp:"nil" controls whether empty input results in a nil pointer. + // nilKind is the kind of empty value allowed for the field. + NilKind NilKind + NilOK bool + + // rlp:"optional" allows for a field to be missing in the input list. + // If this is set, all subsequent fields must also be optional. + Optional bool + + // rlp:"tail" controls whether this field swallows additional list elements. It can + // only be set for the last field, which must be of slice type. + Tail bool + + // rlp:"-" ignores fields. + Ignored bool +} + +// TagError is raised for invalid struct tags. +type TagError struct { + StructType string + + // These are set by this package. + Field string + Tag string + Err string +} + +func (e TagError) Error() string { + field := "field " + e.Field + if e.StructType != "" { + field = e.StructType + "." + e.Field + } + return fmt.Sprintf("rlp: invalid struct tag %q for %s (%s)", e.Tag, field, e.Err) +} + +// ProcessFields filters the given struct fields, returning only fields +// that should be considered for encoding/decoding. +func ProcessFields(allFields []Field) ([]Field, []Tags, error) { + lastPublic := lastPublicField(allFields) + + // Gather all exported fields and their tags. + var fields []Field + var tags []Tags + for _, field := range allFields { + if !field.Exported { + continue + } + ts, err := parseTag(field, lastPublic) + if err != nil { + return nil, nil, err + } + if ts.Ignored { + continue + } + fields = append(fields, field) + tags = append(tags, ts) + } + + // Verify optional field consistency. If any optional field exists, + // all fields after it must also be optional. Note: optional + tail + // is supported. + var anyOptional bool + var firstOptionalName string + for i, ts := range tags { + name := fields[i].Name + if ts.Optional || ts.Tail { + if !anyOptional { + firstOptionalName = name + } + anyOptional = true + } else { + if anyOptional { + msg := fmt.Sprintf("must be optional because preceding field %q is optional", firstOptionalName) + return nil, nil, TagError{Field: name, Err: msg} + } + } + } + return fields, tags, nil +} + +func parseTag(field Field, lastPublic int) (Tags, error) { + name := field.Name + tag := reflect.StructTag(field.Tag) + var ts Tags + for _, t := range strings.Split(tag.Get("rlp"), ",") { + switch t = strings.TrimSpace(t); t { + case "": + // empty tag is allowed for some reason + case "-": + ts.Ignored = true + case "nil", "nilString", "nilList": + ts.NilOK = true + if field.Type.Kind != reflect.Ptr { + return ts, TagError{Field: name, Tag: t, Err: "field is not a pointer"} + } + switch t { + case "nil": + ts.NilKind = field.Type.Elem.DefaultNilValue() + case "nilString": + ts.NilKind = NilKindString + case "nilList": + ts.NilKind = NilKindList + } + case "optional": + ts.Optional = true + if ts.Tail { + return ts, TagError{Field: name, Tag: t, Err: `also has "tail" tag`} + } + case "tail": + ts.Tail = true + if field.Index != lastPublic { + return ts, TagError{Field: name, Tag: t, Err: "must be on last field"} + } + if ts.Optional { + return ts, TagError{Field: name, Tag: t, Err: `also has "optional" tag`} + } + if field.Type.Kind != reflect.Slice { + return ts, TagError{Field: name, Tag: t, Err: "field type is not slice"} + } + default: + return ts, TagError{Field: name, Tag: t, Err: "unknown tag"} + } + } + return ts, nil +} + +func lastPublicField(fields []Field) int { + last := 0 + for _, f := range fields { + if f.Exported { + last = f.Index + } + } + return last +} + +func isUint(k reflect.Kind) bool { + return k >= reflect.Uint && k <= reflect.Uintptr +} + +func isByte(typ Type) bool { + return typ.Kind == reflect.Uint8 && !typ.IsEncoder +} + +func isByteArray(typ Type) bool { + return (typ.Kind == reflect.Slice || typ.Kind == reflect.Array) && isByte(*typ.Elem) +} diff --git a/encoding/rlp/iterator.go b/encoding/rlp/iterator.go new file mode 100644 index 000000000..353ef09fb --- /dev/null +++ b/encoding/rlp/iterator.go @@ -0,0 +1,61 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rlp + +type listIterator struct { + data []byte + next []byte + err error +} + +// NewListIterator creates an iterator for the (list) represented by data +// TODO: Consider removing this implementation, as it is no longer used. +func NewListIterator(data RawValue) (*listIterator, error) { + k, t, c, err := readKind(data) + if err != nil { + return nil, err + } + if k != List { + return nil, ErrExpectedList + } + it := &listIterator{ + data: data[t : t+c], + } + return it, nil + +} + +// Next forwards the iterator one step, returns true if it was not at end yet +func (it *listIterator) Next() bool { + if len(it.data) == 0 { + return false + } + _, t, c, err := readKind(it.data) + it.next = it.data[:t+c] + it.data = it.data[t+c:] + it.err = err + return true +} + +// Value returns the current value +func (it *listIterator) Value() []byte { + return it.next +} + +func (it *listIterator) Err() error { + return it.err +} diff --git a/encoding/rlp/iterator_test.go b/encoding/rlp/iterator_test.go new file mode 100644 index 000000000..aeed9428d --- /dev/null +++ b/encoding/rlp/iterator_test.go @@ -0,0 +1,59 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rlp + +import ( + "testing" + + "github.com/onflow/flow-go-sdk/encoding/common/hexutil" +) + +// TestIterator tests some basic things about the ListIterator. A more +// comprehensive test can be found in core/rlp_test.go, where we can +// use both types and rlp without dependency cycles +func TestIterator(t *testing.T) { + bodyRlpHex := "0xf902cbf8d6f869800182c35094000000000000000000000000000000000000aaaa808a000000000000000000001ba01025c66fad28b4ce3370222624d952c35529e602af7cbe04f667371f61b0e3b3a00ab8813514d1217059748fd903288ace1b4001a4bc5fbde2790debdc8167de2ff869010182c35094000000000000000000000000000000000000aaaa808a000000000000000000001ca05ac4cf1d19be06f3742c21df6c49a7e929ceb3dbaf6a09f3cfb56ff6828bd9a7a06875970133a35e63ac06d360aa166d228cc013e9b96e0a2cae7f55b22e1ee2e8f901f0f901eda0c75448377c0e426b8017b23c5f77379ecf69abc1d5c224284ad3ba1c46c59adaa00000000000000000000000000000000000000000000000000000000000000000940000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000000b9010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000808080808080a00000000000000000000000000000000000000000000000000000000000000000880000000000000000" + bodyRlp := hexutil.MustDecode(bodyRlpHex) + + it, err := NewListIterator(bodyRlp) + if err != nil { + t.Fatal(err) + } + // Check that txs exist + if !it.Next() { + t.Fatal("expected two elems, got zero") + } + txs := it.Value() + // Check that uncles exist + if !it.Next() { + t.Fatal("expected two elems, got one") + } + txit, err := NewListIterator(txs) + if err != nil { + t.Fatal(err) + } + var i = 0 + for txit.Next() { + if txit.err != nil { + t.Fatal(txit.err) + } + i++ + } + if exp := 2; i != exp { + t.Errorf("count wrong, expected %d got %d", i, exp) + } +} diff --git a/encoding/rlp/raw.go b/encoding/rlp/raw.go new file mode 100644 index 000000000..f355efc14 --- /dev/null +++ b/encoding/rlp/raw.go @@ -0,0 +1,261 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rlp + +import ( + "io" + "reflect" +) + +// RawValue represents an encoded RLP value and can be used to delay +// RLP decoding or to precompute an encoding. Note that the decoder does +// not verify whether the content of RawValues is valid RLP. +type RawValue []byte + +var rawValueType = reflect.TypeOf(RawValue{}) + +// ListSize returns the encoded size of an RLP list with the given +// content size. +func ListSize(contentSize uint64) uint64 { + return uint64(headsize(contentSize)) + contentSize +} + +// IntSize returns the encoded size of the integer x. +func IntSize(x uint64) int { + if x < 0x80 { + return 1 + } + return 1 + intsize(x) +} + +// Split returns the content of first RLP value and any +// bytes after the value as subslices of b. +func Split(b []byte) (k Kind, content, rest []byte, err error) { + k, ts, cs, err := readKind(b) + if err != nil { + return 0, nil, b, err + } + return k, b[ts : ts+cs], b[ts+cs:], nil +} + +// SplitString splits b into the content of an RLP string +// and any remaining bytes after the string. +func SplitString(b []byte) (content, rest []byte, err error) { + k, content, rest, err := Split(b) + if err != nil { + return nil, b, err + } + if k == List { + return nil, b, ErrExpectedString + } + return content, rest, nil +} + +// SplitUint64 decodes an integer at the beginning of b. +// It also returns the remaining data after the integer in 'rest'. +func SplitUint64(b []byte) (x uint64, rest []byte, err error) { + content, rest, err := SplitString(b) + if err != nil { + return 0, b, err + } + switch { + case len(content) == 0: + return 0, rest, nil + case len(content) == 1: + if content[0] == 0 { + return 0, b, ErrCanonInt + } + return uint64(content[0]), rest, nil + case len(content) > 8: + return 0, b, errUintOverflow + default: + x, err = readSize(content, byte(len(content))) + if err != nil { + return 0, b, ErrCanonInt + } + return x, rest, nil + } +} + +// SplitList splits b into the content of a list and any remaining +// bytes after the list. +func SplitList(b []byte) (content, rest []byte, err error) { + k, content, rest, err := Split(b) + if err != nil { + return nil, b, err + } + if k != List { + return nil, b, ErrExpectedList + } + return content, rest, nil +} + +// CountValues counts the number of encoded values in b. +func CountValues(b []byte) (int, error) { + i := 0 + for ; len(b) > 0; i++ { + _, tagsize, size, err := readKind(b) + if err != nil { + return 0, err + } + b = b[tagsize+size:] + } + return i, nil +} + +func readKind(buf []byte) (k Kind, tagsize, contentsize uint64, err error) { + if len(buf) == 0 { + return 0, 0, 0, io.ErrUnexpectedEOF + } + b := buf[0] + switch { + case b < 0x80: + k = Byte + tagsize = 0 + contentsize = 1 + case b < 0xB8: + k = String + tagsize = 1 + contentsize = uint64(b - 0x80) + // Reject strings that should've been single bytes. + if contentsize == 1 && len(buf) > 1 && buf[1] < 128 { + return 0, 0, 0, ErrCanonSize + } + case b < 0xC0: + k = String + tagsize = uint64(b-0xB7) + 1 + contentsize, err = readSize(buf[1:], b-0xB7) + case b < 0xF8: + k = List + tagsize = 1 + contentsize = uint64(b - 0xC0) + default: + k = List + tagsize = uint64(b-0xF7) + 1 + contentsize, err = readSize(buf[1:], b-0xF7) + } + if err != nil { + return 0, 0, 0, err + } + // Reject values larger than the input slice. + if contentsize > uint64(len(buf))-tagsize { + return 0, 0, 0, ErrValueTooLarge + } + return k, tagsize, contentsize, err +} + +func readSize(b []byte, slen byte) (uint64, error) { + if int(slen) > len(b) { + return 0, io.ErrUnexpectedEOF + } + var s uint64 + switch slen { + case 1: + s = uint64(b[0]) + case 2: + s = uint64(b[0])<<8 | uint64(b[1]) + case 3: + s = uint64(b[0])<<16 | uint64(b[1])<<8 | uint64(b[2]) + case 4: + s = uint64(b[0])<<24 | uint64(b[1])<<16 | uint64(b[2])<<8 | uint64(b[3]) + case 5: + s = uint64(b[0])<<32 | uint64(b[1])<<24 | uint64(b[2])<<16 | uint64(b[3])<<8 | uint64(b[4]) + case 6: + s = uint64(b[0])<<40 | uint64(b[1])<<32 | uint64(b[2])<<24 | uint64(b[3])<<16 | uint64(b[4])<<8 | uint64(b[5]) + case 7: + s = uint64(b[0])<<48 | uint64(b[1])<<40 | uint64(b[2])<<32 | uint64(b[3])<<24 | uint64(b[4])<<16 | uint64(b[5])<<8 | uint64(b[6]) + case 8: + s = uint64(b[0])<<56 | uint64(b[1])<<48 | uint64(b[2])<<40 | uint64(b[3])<<32 | uint64(b[4])<<24 | uint64(b[5])<<16 | uint64(b[6])<<8 | uint64(b[7]) + } + // Reject sizes < 56 (shouldn't have separate size) and sizes with + // leading zero bytes. + if s < 56 || b[0] == 0 { + return 0, ErrCanonSize + } + return s, nil +} + +// AppendUint64 appends the RLP encoding of i to b, and returns the resulting slice. +func AppendUint64(b []byte, i uint64) []byte { + if i == 0 { + return append(b, 0x80) + } else if i < 128 { + return append(b, byte(i)) + } + switch { + case i < (1 << 8): + return append(b, 0x81, byte(i)) + case i < (1 << 16): + return append(b, 0x82, + byte(i>>8), + byte(i), + ) + case i < (1 << 24): + return append(b, 0x83, + byte(i>>16), + byte(i>>8), + byte(i), + ) + case i < (1 << 32): + return append(b, 0x84, + byte(i>>24), + byte(i>>16), + byte(i>>8), + byte(i), + ) + case i < (1 << 40): + return append(b, 0x85, + byte(i>>32), + byte(i>>24), + byte(i>>16), + byte(i>>8), + byte(i), + ) + + case i < (1 << 48): + return append(b, 0x86, + byte(i>>40), + byte(i>>32), + byte(i>>24), + byte(i>>16), + byte(i>>8), + byte(i), + ) + case i < (1 << 56): + return append(b, 0x87, + byte(i>>48), + byte(i>>40), + byte(i>>32), + byte(i>>24), + byte(i>>16), + byte(i>>8), + byte(i), + ) + + default: + return append(b, 0x88, + byte(i>>56), + byte(i>>48), + byte(i>>40), + byte(i>>32), + byte(i>>24), + byte(i>>16), + byte(i>>8), + byte(i), + ) + } +} diff --git a/encoding/rlp/raw_test.go b/encoding/rlp/raw_test.go new file mode 100644 index 000000000..46adff22c --- /dev/null +++ b/encoding/rlp/raw_test.go @@ -0,0 +1,285 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rlp + +import ( + "bytes" + "errors" + "io" + "testing" + "testing/quick" +) + +func TestCountValues(t *testing.T) { + tests := []struct { + input string // note: spaces in input are stripped by unhex + count int + err error + }{ + // simple cases + {"", 0, nil}, + {"00", 1, nil}, + {"80", 1, nil}, + {"C0", 1, nil}, + {"01 02 03", 3, nil}, + {"01 C406070809 02", 3, nil}, + {"820101 820202 8403030303 04", 4, nil}, + + // size errors + {"8142", 0, ErrCanonSize}, + {"01 01 8142", 0, ErrCanonSize}, + {"02 84020202", 0, ErrValueTooLarge}, + + { + input: "A12000BF49F440A1CD0527E4D06E2765654C0F56452257516D793A9B8D604DCFDF2AB853F851808D10000000000000000000000000A056E81F171BCC55A6FF8345E692C0F86E5B48E01B996CADC001622FB5E363B421A0C5D2460186F7233C927E7DB2DCC703C0E500B653CA82273B7BFAD8045D85A470", + count: 2, + }, + } + for i, test := range tests { + count, err := CountValues(unhex(test.input)) + if count != test.count { + t.Errorf("test %d: count mismatch, got %d want %d\ninput: %s", i, count, test.count, test.input) + } + if !errors.Is(err, test.err) { + t.Errorf("test %d: err mismatch, got %q want %q\ninput: %s", i, err, test.err, test.input) + } + } +} + +func TestSplitTypes(t *testing.T) { + if _, _, err := SplitString(unhex("C100")); err != ErrExpectedString { + t.Errorf("SplitString returned %q, want %q", err, ErrExpectedString) + } + if _, _, err := SplitList(unhex("01")); err != ErrExpectedList { + t.Errorf("SplitString returned %q, want %q", err, ErrExpectedList) + } + if _, _, err := SplitList(unhex("81FF")); err != ErrExpectedList { + t.Errorf("SplitString returned %q, want %q", err, ErrExpectedList) + } +} + +func TestSplitUint64(t *testing.T) { + tests := []struct { + input string + val uint64 + rest string + err error + }{ + {"01", 1, "", nil}, + {"7FFF", 0x7F, "FF", nil}, + {"80FF", 0, "FF", nil}, + {"81FAFF", 0xFA, "FF", nil}, + {"82FAFAFF", 0xFAFA, "FF", nil}, + {"83FAFAFAFF", 0xFAFAFA, "FF", nil}, + {"84FAFAFAFAFF", 0xFAFAFAFA, "FF", nil}, + {"85FAFAFAFAFAFF", 0xFAFAFAFAFA, "FF", nil}, + {"86FAFAFAFAFAFAFF", 0xFAFAFAFAFAFA, "FF", nil}, + {"87FAFAFAFAFAFAFAFF", 0xFAFAFAFAFAFAFA, "FF", nil}, + {"88FAFAFAFAFAFAFAFAFF", 0xFAFAFAFAFAFAFAFA, "FF", nil}, + + // errors + {"", 0, "", io.ErrUnexpectedEOF}, + {"00", 0, "00", ErrCanonInt}, + {"81", 0, "81", ErrValueTooLarge}, + {"8100", 0, "8100", ErrCanonSize}, + {"8200FF", 0, "8200FF", ErrCanonInt}, + {"8103FF", 0, "8103FF", ErrCanonSize}, + {"89FAFAFAFAFAFAFAFAFAFF", 0, "89FAFAFAFAFAFAFAFAFAFF", errUintOverflow}, + } + + for i, test := range tests { + val, rest, err := SplitUint64(unhex(test.input)) + if val != test.val { + t.Errorf("test %d: val mismatch: got %x, want %x (input %q)", i, val, test.val, test.input) + } + if !bytes.Equal(rest, unhex(test.rest)) { + t.Errorf("test %d: rest mismatch: got %x, want %s (input %q)", i, rest, test.rest, test.input) + } + if err != test.err { + t.Errorf("test %d: error mismatch: got %q, want %q", i, err, test.err) + } + } +} + +func TestSplit(t *testing.T) { + tests := []struct { + input string + kind Kind + val, rest string + err error + }{ + {input: "00FFFF", kind: Byte, val: "00", rest: "FFFF"}, + {input: "01FFFF", kind: Byte, val: "01", rest: "FFFF"}, + {input: "7FFFFF", kind: Byte, val: "7F", rest: "FFFF"}, + {input: "80FFFF", kind: String, val: "", rest: "FFFF"}, + {input: "C3010203", kind: List, val: "010203"}, + + // errors + {input: "", err: io.ErrUnexpectedEOF}, + + {input: "8141", err: ErrCanonSize, rest: "8141"}, + {input: "B800", err: ErrCanonSize, rest: "B800"}, + {input: "B802FFFF", err: ErrCanonSize, rest: "B802FFFF"}, + {input: "B90000", err: ErrCanonSize, rest: "B90000"}, + {input: "B90055", err: ErrCanonSize, rest: "B90055"}, + {input: "BA0002FFFF", err: ErrCanonSize, rest: "BA0002FFFF"}, + {input: "F800", err: ErrCanonSize, rest: "F800"}, + {input: "F90000", err: ErrCanonSize, rest: "F90000"}, + {input: "F90055", err: ErrCanonSize, rest: "F90055"}, + {input: "FA0002FFFF", err: ErrCanonSize, rest: "FA0002FFFF"}, + + {input: "81", err: ErrValueTooLarge, rest: "81"}, + {input: "8501010101", err: ErrValueTooLarge, rest: "8501010101"}, + {input: "C60607080902", err: ErrValueTooLarge, rest: "C60607080902"}, + + // size check overflow + {input: "BFFFFFFFFFFFFFFFFF", err: ErrValueTooLarge, rest: "BFFFFFFFFFFFFFFFFF"}, + {input: "FFFFFFFFFFFFFFFFFF", err: ErrValueTooLarge, rest: "FFFFFFFFFFFFFFFFFF"}, + + { + input: "B838FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF", + err: ErrValueTooLarge, + rest: "B838FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF", + }, + { + input: "F838FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF", + err: ErrValueTooLarge, + rest: "F838FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF", + }, + + // a few bigger values, just for kicks + { + input: "F839FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF", + kind: List, + val: "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF", + rest: "", + }, + { + input: "F90211A060EF29F20CC1007AE6E9530AEE16F4B31F8F1769A2D1264EC995C6D1241868D6A07C62AB8AC9838F5F5877B20BB37B387BC2106E97A3D52172CBEDB5EE17C36008A00EAB6B7324AADC0F6047C6AFC8229F09F7CF451B51D67C8DFB08D49BA8C3C626A04453343B2F3A6E42FCF87948F88AF7C8FC16D0C2735CBA7F026836239AB2C15FA024635C7291C882CE4C0763760C1A362DFC3FFCD802A55722236DE058D74202ACA0A220C808DE10F55E40AB25255201CFF009EA181D3906638E944EE2BF34049984A08D325AB26796F1CCB470F69C0F842501DC35D368A0C2575B2D243CFD1E8AB0FDA0B5298FF60DA5069463D610513C9F04F24051348391A143AFFAB7197DFACDEA72A02D2A7058A4463F8FB69378369E11EF33AE3252E2DB86CB545B36D3C26DDECE5AA0888F97BCA8E0BD83DC5B3B91CFF5FAF2F66F9501010682D67EF4A3B4E66115FBA0E8175A60C93BE9ED02921958F0EA55DA0FB5E4802AF5846147BAD92BC2D8AF26A08B3376FF433F3A4250FA64B7F804004CAC5807877D91C4427BD1CD05CF912ED8A09B32EF0F03BD13C37FF950C0CCCEFCCDD6669F2E7F2AA5CB859928E84E29763EA09BBA5E46610C8C8B1F8E921E5691BF8C7E40D75825D5EA3217AA9C3A8A355F39A0EEB95BC78251CCCEC54A97F19755C4A59A293544EEE6119AFA50531211E53C4FA00B6E86FE150BF4A9E0FEEE9C90F5465E617A861BB5E357F942881EE762212E2580", + kind: List, + val: "A060EF29F20CC1007AE6E9530AEE16F4B31F8F1769A2D1264EC995C6D1241868D6A07C62AB8AC9838F5F5877B20BB37B387BC2106E97A3D52172CBEDB5EE17C36008A00EAB6B7324AADC0F6047C6AFC8229F09F7CF451B51D67C8DFB08D49BA8C3C626A04453343B2F3A6E42FCF87948F88AF7C8FC16D0C2735CBA7F026836239AB2C15FA024635C7291C882CE4C0763760C1A362DFC3FFCD802A55722236DE058D74202ACA0A220C808DE10F55E40AB25255201CFF009EA181D3906638E944EE2BF34049984A08D325AB26796F1CCB470F69C0F842501DC35D368A0C2575B2D243CFD1E8AB0FDA0B5298FF60DA5069463D610513C9F04F24051348391A143AFFAB7197DFACDEA72A02D2A7058A4463F8FB69378369E11EF33AE3252E2DB86CB545B36D3C26DDECE5AA0888F97BCA8E0BD83DC5B3B91CFF5FAF2F66F9501010682D67EF4A3B4E66115FBA0E8175A60C93BE9ED02921958F0EA55DA0FB5E4802AF5846147BAD92BC2D8AF26A08B3376FF433F3A4250FA64B7F804004CAC5807877D91C4427BD1CD05CF912ED8A09B32EF0F03BD13C37FF950C0CCCEFCCDD6669F2E7F2AA5CB859928E84E29763EA09BBA5E46610C8C8B1F8E921E5691BF8C7E40D75825D5EA3217AA9C3A8A355F39A0EEB95BC78251CCCEC54A97F19755C4A59A293544EEE6119AFA50531211E53C4FA00B6E86FE150BF4A9E0FEEE9C90F5465E617A861BB5E357F942881EE762212E2580", + rest: "", + }, + { + input: "F877A12000BF49F440A1CD0527E4D06E2765654C0F56452257516D793A9B8D604DCFDF2AB853F851808D10000000000000000000000000A056E81F171BCC55A6FF8345E692C0F86E5B48E01B996CADC001622FB5E363B421A0C5D2460186F7233C927E7DB2DCC703C0E500B653CA82273B7BFAD8045D85A470", + kind: List, + val: "A12000BF49F440A1CD0527E4D06E2765654C0F56452257516D793A9B8D604DCFDF2AB853F851808D10000000000000000000000000A056E81F171BCC55A6FF8345E692C0F86E5B48E01B996CADC001622FB5E363B421A0C5D2460186F7233C927E7DB2DCC703C0E500B653CA82273B7BFAD8045D85A470", + rest: "", + }, + } + + for i, test := range tests { + kind, val, rest, err := Split(unhex(test.input)) + if kind != test.kind { + t.Errorf("test %d: kind mismatch: got %v, want %v", i, kind, test.kind) + } + if !bytes.Equal(val, unhex(test.val)) { + t.Errorf("test %d: val mismatch: got %x, want %s", i, val, test.val) + } + if !bytes.Equal(rest, unhex(test.rest)) { + t.Errorf("test %d: rest mismatch: got %x, want %s", i, rest, test.rest) + } + if err != test.err { + t.Errorf("test %d: error mismatch: got %q, want %q", i, err, test.err) + } + } +} + +func TestReadSize(t *testing.T) { + tests := []struct { + input string + slen byte + size uint64 + err error + }{ + {input: "", slen: 1, err: io.ErrUnexpectedEOF}, + {input: "FF", slen: 2, err: io.ErrUnexpectedEOF}, + {input: "00", slen: 1, err: ErrCanonSize}, + {input: "36", slen: 1, err: ErrCanonSize}, + {input: "37", slen: 1, err: ErrCanonSize}, + {input: "38", slen: 1, size: 0x38}, + {input: "FF", slen: 1, size: 0xFF}, + {input: "FFFF", slen: 2, size: 0xFFFF}, + {input: "FFFFFF", slen: 3, size: 0xFFFFFF}, + {input: "FFFFFFFF", slen: 4, size: 0xFFFFFFFF}, + {input: "FFFFFFFFFF", slen: 5, size: 0xFFFFFFFFFF}, + {input: "FFFFFFFFFFFF", slen: 6, size: 0xFFFFFFFFFFFF}, + {input: "FFFFFFFFFFFFFF", slen: 7, size: 0xFFFFFFFFFFFFFF}, + {input: "FFFFFFFFFFFFFFFF", slen: 8, size: 0xFFFFFFFFFFFFFFFF}, + {input: "0102", slen: 2, size: 0x0102}, + {input: "010203", slen: 3, size: 0x010203}, + {input: "01020304", slen: 4, size: 0x01020304}, + {input: "0102030405", slen: 5, size: 0x0102030405}, + {input: "010203040506", slen: 6, size: 0x010203040506}, + {input: "01020304050607", slen: 7, size: 0x01020304050607}, + {input: "0102030405060708", slen: 8, size: 0x0102030405060708}, + } + + for _, test := range tests { + size, err := readSize(unhex(test.input), test.slen) + if err != test.err { + t.Errorf("readSize(%s, %d): error mismatch: got %q, want %q", test.input, test.slen, err, test.err) + continue + } + if size != test.size { + t.Errorf("readSize(%s, %d): size mismatch: got %#x, want %#x", test.input, test.slen, size, test.size) + } + } +} + +func TestAppendUint64(t *testing.T) { + tests := []struct { + input uint64 + slice []byte + output string + }{ + {0, nil, "80"}, + {1, nil, "01"}, + {2, nil, "02"}, + {127, nil, "7F"}, + {128, nil, "8180"}, + {129, nil, "8181"}, + {0xFFFFFF, nil, "83FFFFFF"}, + {127, []byte{1, 2, 3}, "0102037F"}, + {0xFFFFFF, []byte{1, 2, 3}, "01020383FFFFFF"}, + } + + for _, test := range tests { + x := AppendUint64(test.slice, test.input) + if !bytes.Equal(x, unhex(test.output)) { + t.Errorf("AppendUint64(%v, %d): got %x, want %s", test.slice, test.input, x, test.output) + } + + // Check that IntSize returns the appended size. + length := len(x) - len(test.slice) + if s := IntSize(test.input); s != length { + t.Errorf("IntSize(%d): got %d, want %d", test.input, s, length) + } + } +} + +func TestAppendUint64Random(t *testing.T) { + fn := func(i uint64) bool { + enc, _ := EncodeToBytes(i) + encAppend := AppendUint64(nil, i) + return bytes.Equal(enc, encAppend) + } + config := quick.Config{MaxCountScale: 50} + if err := quick.Check(fn, &config); err != nil { + t.Fatal(err) + } +} diff --git a/encoding/rlp/rlpgen/gen.go b/encoding/rlp/rlpgen/gen.go new file mode 100644 index 000000000..c41317408 --- /dev/null +++ b/encoding/rlp/rlpgen/gen.go @@ -0,0 +1,751 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package main + +import ( + "bytes" + "fmt" + "go/format" + "go/types" + "sort" + + "github.com/onflow/flow-go-sdk/encoding/rlp/internal/rlpstruct" +) + +// buildContext keeps the data needed for make*Op. +type buildContext struct { + topType *types.Named // the type we're creating methods for + + encoderIface *types.Interface + decoderIface *types.Interface + rawValueType *types.Named + + typeToStructCache map[types.Type]*rlpstruct.Type +} + +func newBuildContext(packageRLP *types.Package) *buildContext { + enc := packageRLP.Scope().Lookup("Encoder").Type().Underlying() + dec := packageRLP.Scope().Lookup("Decoder").Type().Underlying() + rawv := packageRLP.Scope().Lookup("RawValue").Type() + return &buildContext{ + typeToStructCache: make(map[types.Type]*rlpstruct.Type), + encoderIface: enc.(*types.Interface), + decoderIface: dec.(*types.Interface), + rawValueType: rawv.(*types.Named), + } +} + +func (bctx *buildContext) isEncoder(typ types.Type) bool { + return types.Implements(typ, bctx.encoderIface) +} + +func (bctx *buildContext) isDecoder(typ types.Type) bool { + return types.Implements(typ, bctx.decoderIface) +} + +// typeToStructType converts typ to rlpstruct.Type. +func (bctx *buildContext) typeToStructType(typ types.Type) *rlpstruct.Type { + if prev := bctx.typeToStructCache[typ]; prev != nil { + return prev // short-circuit for recursive types. + } + + // Resolve named types to their underlying type, but keep the name. + name := types.TypeString(typ, nil) + for { + utype := typ.Underlying() + if utype == typ { + break + } + typ = utype + } + + // Create the type and store it in cache. + t := &rlpstruct.Type{ + Name: name, + Kind: typeReflectKind(typ), + IsEncoder: bctx.isEncoder(typ), + IsDecoder: bctx.isDecoder(typ), + } + bctx.typeToStructCache[typ] = t + + // Assign element type. + switch typ.(type) { + case *types.Array, *types.Slice, *types.Pointer: + etype := typ.(interface{ Elem() types.Type }).Elem() + t.Elem = bctx.typeToStructType(etype) + } + return t +} + +// genContext is passed to the gen* methods of op when generating +// the output code. It tracks packages to be imported by the output +// file and assigns unique names of temporary variables. +type genContext struct { + inPackage *types.Package + imports map[string]struct{} + tempCounter int +} + +func newGenContext(inPackage *types.Package) *genContext { + return &genContext{ + inPackage: inPackage, + imports: make(map[string]struct{}), + } +} + +func (ctx *genContext) temp() string { + v := fmt.Sprintf("_tmp%d", ctx.tempCounter) + ctx.tempCounter++ + return v +} + +func (ctx *genContext) resetTemp() { + ctx.tempCounter = 0 +} + +func (ctx *genContext) addImport(path string) { + if path == ctx.inPackage.Path() { + return // avoid importing the package that we're generating in. + } + // TODO: renaming? + ctx.imports[path] = struct{}{} +} + +// importsList returns all packages that need to be imported. +func (ctx *genContext) importsList() []string { + imp := make([]string, 0, len(ctx.imports)) + for k := range ctx.imports { + imp = append(imp, k) + } + sort.Strings(imp) + return imp +} + +// qualify is the types.Qualifier used for printing types. +func (ctx *genContext) qualify(pkg *types.Package) string { + if pkg.Path() == ctx.inPackage.Path() { + return "" + } + ctx.addImport(pkg.Path()) + // TODO: renaming? + return pkg.Name() +} + +type op interface { + // genWrite creates the encoder. The generated code should write v, + // which is any Go expression, to the rlp.EncoderBuffer 'w'. + genWrite(ctx *genContext, v string) string + + // genDecode creates the decoder. The generated code should read + // a value from the rlp.Stream 'dec' and store it to dst. + genDecode(ctx *genContext) (string, string) +} + +// basicOp handles basic types bool, uint*, string. +type basicOp struct { + typ types.Type + writeMethod string // calle write the value + writeArgType types.Type // parameter type of writeMethod + decMethod string + decResultType types.Type // return type of decMethod + decUseBitSize bool // if true, result bit size is appended to decMethod +} + +func (*buildContext) makeBasicOp(typ *types.Basic) (op, error) { + op := basicOp{typ: typ} + kind := typ.Kind() + switch { + case kind == types.Bool: + op.writeMethod = "WriteBool" + op.writeArgType = types.Typ[types.Bool] + op.decMethod = "Bool" + op.decResultType = types.Typ[types.Bool] + case kind >= types.Uint8 && kind <= types.Uint64: + op.writeMethod = "WriteUint64" + op.writeArgType = types.Typ[types.Uint64] + op.decMethod = "Uint" + op.decResultType = typ + op.decUseBitSize = true + case kind == types.String: + op.writeMethod = "WriteString" + op.writeArgType = types.Typ[types.String] + op.decMethod = "String" + op.decResultType = types.Typ[types.String] + default: + return nil, fmt.Errorf("unhandled basic type: %v", typ) + } + return op, nil +} + +func (*buildContext) makeByteSliceOp(typ *types.Slice) op { + if !isByte(typ.Elem()) { + panic("non-byte slice type in makeByteSliceOp") + } + bslice := types.NewSlice(types.Typ[types.Uint8]) + return basicOp{ + typ: typ, + writeMethod: "WriteBytes", + writeArgType: bslice, + decMethod: "Bytes", + decResultType: bslice, + } +} + +func (bctx *buildContext) makeRawValueOp() op { + bslice := types.NewSlice(types.Typ[types.Uint8]) + return basicOp{ + typ: bctx.rawValueType, + writeMethod: "Write", + writeArgType: bslice, + decMethod: "Raw", + decResultType: bslice, + } +} + +func (op basicOp) writeNeedsConversion() bool { + return !types.AssignableTo(op.typ, op.writeArgType) +} + +func (op basicOp) decodeNeedsConversion() bool { + return !types.AssignableTo(op.decResultType, op.typ) +} + +func (op basicOp) genWrite(ctx *genContext, v string) string { + if op.writeNeedsConversion() { + v = fmt.Sprintf("%s(%s)", op.writeArgType, v) + } + return fmt.Sprintf("w.%s(%s)\n", op.writeMethod, v) +} + +func (op basicOp) genDecode(ctx *genContext) (string, string) { + var ( + resultV = ctx.temp() + result = resultV + method = op.decMethod + ) + if op.decUseBitSize { + // Note: For now, this only works for platform-independent integer + // sizes. makeBasicOp forbids the platform-dependent types. + var sizes types.StdSizes + method = fmt.Sprintf("%s%d", op.decMethod, sizes.Sizeof(op.typ)*8) + } + + // Call the decoder method. + var b bytes.Buffer + fmt.Fprintf(&b, "%s, err := dec.%s()\n", resultV, method) + fmt.Fprintf(&b, "if err != nil { return err }\n") + if op.decodeNeedsConversion() { + conv := ctx.temp() + fmt.Fprintf(&b, "%s := %s(%s)\n", conv, types.TypeString(op.typ, ctx.qualify), resultV) + result = conv + } + return result, b.String() +} + +// byteArrayOp handles [...]byte. +type byteArrayOp struct { + typ types.Type + name types.Type // name != typ for named byte array types (e.g. common.Address) +} + +func (bctx *buildContext) makeByteArrayOp(name *types.Named, typ *types.Array) byteArrayOp { + nt := types.Type(name) + if name == nil { + nt = typ + } + return byteArrayOp{typ, nt} +} + +func (op byteArrayOp) genWrite(ctx *genContext, v string) string { + return fmt.Sprintf("w.WriteBytes(%s[:])\n", v) +} + +func (op byteArrayOp) genDecode(ctx *genContext) (string, string) { + var resultV = ctx.temp() + + var b bytes.Buffer + fmt.Fprintf(&b, "var %s %s\n", resultV, types.TypeString(op.name, ctx.qualify)) + fmt.Fprintf(&b, "if err := dec.ReadBytes(%s[:]); err != nil { return err }\n", resultV) + return resultV, b.String() +} + +// bigIntNoPtrOp handles non-pointer big.Int. +// This exists because big.Int has it's own decoder operation on rlp.Stream, +// but the decode method returns *big.Int, so it needs to be dereferenced. +type bigIntOp struct { + pointer bool +} + +func (op bigIntOp) genWrite(ctx *genContext, v string) string { + var b bytes.Buffer + + fmt.Fprintf(&b, "if %s.Sign() == -1 {\n", v) + fmt.Fprintf(&b, " return rlp.ErrNegativeBigInt\n") + fmt.Fprintf(&b, "}\n") + dst := v + if !op.pointer { + dst = "&" + v + } + fmt.Fprintf(&b, "w.WriteBigInt(%s)\n", dst) + + // Wrap with nil check. + if op.pointer { + code := b.String() + b.Reset() + fmt.Fprintf(&b, "if %s == nil {\n", v) + fmt.Fprintf(&b, " w.Write(rlp.EmptyString)") + fmt.Fprintf(&b, "} else {\n") + fmt.Fprint(&b, code) + fmt.Fprintf(&b, "}\n") + } + + return b.String() +} + +func (op bigIntOp) genDecode(ctx *genContext) (string, string) { + var resultV = ctx.temp() + + var b bytes.Buffer + fmt.Fprintf(&b, "%s, err := dec.BigInt()\n", resultV) + fmt.Fprintf(&b, "if err != nil { return err }\n") + + result := resultV + if !op.pointer { + result = "(*" + resultV + ")" + } + return result, b.String() +} + +// encoderDecoderOp handles rlp.Encoder and rlp.Decoder. +// In order to be used with this, the type must implement both interfaces. +// This restriction may be lifted in the future by creating separate ops for +// encoding and decoding. +type encoderDecoderOp struct { + typ types.Type +} + +func (op encoderDecoderOp) genWrite(ctx *genContext, v string) string { + return fmt.Sprintf("if err := %s.EncodeRLP(w); err != nil { return err }\n", v) +} + +func (op encoderDecoderOp) genDecode(ctx *genContext) (string, string) { + // DecodeRLP must have pointer receiver, and this is verified in makeOp. + etyp := op.typ.(*types.Pointer).Elem() + var resultV = ctx.temp() + + var b bytes.Buffer + fmt.Fprintf(&b, "%s := new(%s)\n", resultV, types.TypeString(etyp, ctx.qualify)) + fmt.Fprintf(&b, "if err := %s.DecodeRLP(dec); err != nil { return err }\n", resultV) + return resultV, b.String() +} + +// ptrOp handles pointer types. +type ptrOp struct { + elemTyp types.Type + elem op + nilOK bool + nilValue rlpstruct.NilKind +} + +func (bctx *buildContext) makePtrOp(elemTyp types.Type, tags rlpstruct.Tags) (op, error) { + elemOp, err := bctx.makeOp(nil, elemTyp, rlpstruct.Tags{}) + if err != nil { + return nil, err + } + op := ptrOp{elemTyp: elemTyp, elem: elemOp} + + // Determine nil value. + if tags.NilOK { + op.nilOK = true + op.nilValue = tags.NilKind + } else { + styp := bctx.typeToStructType(elemTyp) + op.nilValue = styp.DefaultNilValue() + } + return op, nil +} + +func (op ptrOp) genWrite(ctx *genContext, v string) string { + // Note: in writer functions, accesses to v are read-only, i.e. v is any Go + // expression. To make all accesses work through the pointer, we substitute + // v with (*v). This is required for most accesses including `v`, `call(v)`, + // and `v[index]` on slices. + // + // For `v.field` and `v[:]` on arrays, the dereference operation is not required. + var vv string + _, isStruct := op.elem.(structOp) + _, isByteArray := op.elem.(byteArrayOp) + if isStruct || isByteArray { + vv = v + } else { + vv = fmt.Sprintf("(*%s)", v) + } + + var b bytes.Buffer + fmt.Fprintf(&b, "if %s == nil {\n", v) + fmt.Fprintf(&b, " w.Write([]byte{0x%X})\n", op.nilValue) + fmt.Fprintf(&b, "} else {\n") + fmt.Fprintf(&b, " %s", op.elem.genWrite(ctx, vv)) + fmt.Fprintf(&b, "}\n") + return b.String() +} + +func (op ptrOp) genDecode(ctx *genContext) (string, string) { + result, code := op.elem.genDecode(ctx) + if !op.nilOK { + // If nil pointers are not allowed, we can just decode the element. + return "&" + result, code + } + + // nil is allowed, so check the kind and size first. + // If size is zero and kind matches the nilKind of the type, + // the value decodes as a nil pointer. + var ( + resultV = ctx.temp() + kindV = ctx.temp() + sizeV = ctx.temp() + wantKind string + ) + if op.nilValue == rlpstruct.NilKindList { + wantKind = "rlp.List" + } else { + wantKind = "rlp.String" + } + var b bytes.Buffer + fmt.Fprintf(&b, "var %s %s\n", resultV, types.TypeString(types.NewPointer(op.elemTyp), ctx.qualify)) + fmt.Fprintf(&b, "if %s, %s, err := dec.Kind(); err != nil {\n", kindV, sizeV) + fmt.Fprintf(&b, " return err\n") + fmt.Fprintf(&b, "} else if %s != 0 || %s != %s {\n", sizeV, kindV, wantKind) + fmt.Fprint(&b, code) + fmt.Fprintf(&b, " %s = &%s\n", resultV, result) + fmt.Fprintf(&b, "}\n") + return resultV, b.String() +} + +// structOp handles struct types. +type structOp struct { + named *types.Named + typ *types.Struct + fields []*structField + optionalFields []*structField +} + +type structField struct { + name string + typ types.Type + elem op +} + +func (bctx *buildContext) makeStructOp(named *types.Named, typ *types.Struct) (op, error) { + // Convert fields to []rlpstruct.Field. + var allStructFields []rlpstruct.Field + for i := 0; i < typ.NumFields(); i++ { + f := typ.Field(i) + allStructFields = append(allStructFields, rlpstruct.Field{ + Name: f.Name(), + Exported: f.Exported(), + Index: i, + Tag: typ.Tag(i), + Type: *bctx.typeToStructType(f.Type()), + }) + } + + // Filter/validate fields. + fields, tags, err := rlpstruct.ProcessFields(allStructFields) + if err != nil { + return nil, err + } + + // Create field ops. + var op = structOp{named: named, typ: typ} + for i, field := range fields { + // Advanced struct tags are not supported yet. + tag := tags[i] + if err := checkUnsupportedTags(field.Name, tag); err != nil { + return nil, err + } + typ := typ.Field(field.Index).Type() + elem, err := bctx.makeOp(nil, typ, tags[i]) + if err != nil { + return nil, fmt.Errorf("field %s: %v", field.Name, err) + } + f := &structField{name: field.Name, typ: typ, elem: elem} + if tag.Optional { + op.optionalFields = append(op.optionalFields, f) + } else { + op.fields = append(op.fields, f) + } + } + return op, nil +} + +func checkUnsupportedTags(field string, tag rlpstruct.Tags) error { + if tag.Tail { + return fmt.Errorf(`field %s has unsupported struct tag "tail"`, field) + } + return nil +} + +func (op structOp) genWrite(ctx *genContext, v string) string { + var b bytes.Buffer + var listMarker = ctx.temp() + fmt.Fprintf(&b, "%s := w.List()\n", listMarker) + for _, field := range op.fields { + selector := v + "." + field.name + fmt.Fprint(&b, field.elem.genWrite(ctx, selector)) + } + op.writeOptionalFields(&b, ctx, v) + fmt.Fprintf(&b, "w.ListEnd(%s)\n", listMarker) + return b.String() +} + +func (op structOp) writeOptionalFields(b *bytes.Buffer, ctx *genContext, v string) { + if len(op.optionalFields) == 0 { + return + } + // First check zero-ness of all optional fields. + var zeroV = make([]string, len(op.optionalFields)) + for i, field := range op.optionalFields { + selector := v + "." + field.name + zeroV[i] = ctx.temp() + fmt.Fprintf(b, "%s := %s\n", zeroV[i], nonZeroCheck(selector, field.typ, ctx.qualify)) + } + // Now write the fields. + for i, field := range op.optionalFields { + selector := v + "." + field.name + cond := "" + for j := i; j < len(op.optionalFields); j++ { + if j > i { + cond += " || " + } + cond += zeroV[j] + } + fmt.Fprintf(b, "if %s {\n", cond) + fmt.Fprint(b, field.elem.genWrite(ctx, selector)) + fmt.Fprintf(b, "}\n") + } +} + +func (op structOp) genDecode(ctx *genContext) (string, string) { + // Get the string representation of the type. + // Here, named types are handled separately because the output + // would contain a copy of the struct definition otherwise. + var typeName string + if op.named != nil { + typeName = types.TypeString(op.named, ctx.qualify) + } else { + typeName = types.TypeString(op.typ, ctx.qualify) + } + + // Create struct object. + var resultV = ctx.temp() + var b bytes.Buffer + fmt.Fprintf(&b, "var %s %s\n", resultV, typeName) + + // Decode fields. + fmt.Fprintf(&b, "{\n") + fmt.Fprintf(&b, "if _, err := dec.List(); err != nil { return err }\n") + for _, field := range op.fields { + result, code := field.elem.genDecode(ctx) + fmt.Fprintf(&b, "// %s:\n", field.name) + fmt.Fprint(&b, code) + fmt.Fprintf(&b, "%s.%s = %s\n", resultV, field.name, result) + } + op.decodeOptionalFields(&b, ctx, resultV) + fmt.Fprintf(&b, "if err := dec.ListEnd(); err != nil { return err }\n") + fmt.Fprintf(&b, "}\n") + return resultV, b.String() +} + +func (op structOp) decodeOptionalFields(b *bytes.Buffer, ctx *genContext, resultV string) { + var suffix bytes.Buffer + for _, field := range op.optionalFields { + result, code := field.elem.genDecode(ctx) + fmt.Fprintf(b, "// %s:\n", field.name) + fmt.Fprintf(b, "if dec.MoreDataInList() {\n") + fmt.Fprint(b, code) + fmt.Fprintf(b, "%s.%s = %s\n", resultV, field.name, result) + fmt.Fprintf(&suffix, "}\n") + } + suffix.WriteTo(b) +} + +// sliceOp handles slice types. +type sliceOp struct { + typ *types.Slice + elemOp op +} + +func (bctx *buildContext) makeSliceOp(typ *types.Slice) (op, error) { + elemOp, err := bctx.makeOp(nil, typ.Elem(), rlpstruct.Tags{}) + if err != nil { + return nil, err + } + return sliceOp{typ: typ, elemOp: elemOp}, nil +} + +func (op sliceOp) genWrite(ctx *genContext, v string) string { + var ( + listMarker = ctx.temp() // holds return value of w.List() + iterElemV = ctx.temp() // iteration variable + elemCode = op.elemOp.genWrite(ctx, iterElemV) + ) + + var b bytes.Buffer + fmt.Fprintf(&b, "%s := w.List()\n", listMarker) + fmt.Fprintf(&b, "for _, %s := range %s {\n", iterElemV, v) + fmt.Fprint(&b, elemCode) + fmt.Fprintf(&b, "}\n") + fmt.Fprintf(&b, "w.ListEnd(%s)\n", listMarker) + return b.String() +} + +func (op sliceOp) genDecode(ctx *genContext) (string, string) { + var sliceV = ctx.temp() // holds the output slice + elemResult, elemCode := op.elemOp.genDecode(ctx) + + var b bytes.Buffer + fmt.Fprintf(&b, "var %s %s\n", sliceV, types.TypeString(op.typ, ctx.qualify)) + fmt.Fprintf(&b, "if _, err := dec.List(); err != nil { return err }\n") + fmt.Fprintf(&b, "for dec.MoreDataInList() {\n") + fmt.Fprintf(&b, " %s", elemCode) + fmt.Fprintf(&b, " %s = append(%s, %s)\n", sliceV, sliceV, elemResult) + fmt.Fprintf(&b, "}\n") + fmt.Fprintf(&b, "if err := dec.ListEnd(); err != nil { return err }\n") + return sliceV, b.String() +} + +func (bctx *buildContext) makeOp(name *types.Named, typ types.Type, tags rlpstruct.Tags) (op, error) { + switch typ := typ.(type) { + case *types.Named: + if isBigInt(typ) { + return bigIntOp{}, nil + } + if typ == bctx.rawValueType { + return bctx.makeRawValueOp(), nil + } + if bctx.isDecoder(typ) { + return nil, fmt.Errorf("type %v implements rlp.Decoder with non-pointer receiver", typ) + } + // TODO: same check for encoder? + return bctx.makeOp(typ, typ.Underlying(), tags) + case *types.Pointer: + if isBigInt(typ.Elem()) { + return bigIntOp{pointer: true}, nil + } + // Encoder/Decoder interfaces. + if bctx.isEncoder(typ) { + if bctx.isDecoder(typ) { + return encoderDecoderOp{typ}, nil + } + return nil, fmt.Errorf("type %v implements rlp.Encoder but not rlp.Decoder", typ) + } + if bctx.isDecoder(typ) { + return nil, fmt.Errorf("type %v implements rlp.Decoder but not rlp.Encoder", typ) + } + // Default pointer handling. + return bctx.makePtrOp(typ.Elem(), tags) + case *types.Basic: + return bctx.makeBasicOp(typ) + case *types.Struct: + return bctx.makeStructOp(name, typ) + case *types.Slice: + etyp := typ.Elem() + if isByte(etyp) && !bctx.isEncoder(etyp) { + return bctx.makeByteSliceOp(typ), nil + } + return bctx.makeSliceOp(typ) + case *types.Array: + etyp := typ.Elem() + if isByte(etyp) && !bctx.isEncoder(etyp) { + return bctx.makeByteArrayOp(name, typ), nil + } + return nil, fmt.Errorf("unhandled array type: %v", typ) + default: + return nil, fmt.Errorf("unhandled type: %v", typ) + } +} + +// generateDecoder generates the DecodeRLP method on 'typ'. +func generateDecoder(ctx *genContext, typ string, op op) []byte { + ctx.resetTemp() + ctx.addImport(pathOfPackageRLP) + + result, code := op.genDecode(ctx) + var b bytes.Buffer + fmt.Fprintf(&b, "func (obj *%s) DecodeRLP(dec *rlp.Stream) error {\n", typ) + fmt.Fprint(&b, code) + fmt.Fprintf(&b, " *obj = %s\n", result) + fmt.Fprintf(&b, " return nil\n") + fmt.Fprintf(&b, "}\n") + return b.Bytes() +} + +// generateEncoder generates the EncodeRLP method on 'typ'. +func generateEncoder(ctx *genContext, typ string, op op) []byte { + ctx.resetTemp() + ctx.addImport("io") + ctx.addImport(pathOfPackageRLP) + + var b bytes.Buffer + fmt.Fprintf(&b, "func (obj *%s) EncodeRLP(_w io.Writer) error {\n", typ) + fmt.Fprintf(&b, " w := rlp.NewEncoderBuffer(_w)\n") + fmt.Fprint(&b, op.genWrite(ctx, "obj")) + fmt.Fprintf(&b, " return w.Flush()\n") + fmt.Fprintf(&b, "}\n") + return b.Bytes() +} + +func (bctx *buildContext) generate(typ *types.Named, encoder, decoder bool) ([]byte, error) { + bctx.topType = typ + + pkg := typ.Obj().Pkg() + op, err := bctx.makeOp(nil, typ, rlpstruct.Tags{}) + if err != nil { + return nil, err + } + + var ( + ctx = newGenContext(pkg) + encSource []byte + decSource []byte + ) + if encoder { + encSource = generateEncoder(ctx, typ.Obj().Name(), op) + } + if decoder { + decSource = generateDecoder(ctx, typ.Obj().Name(), op) + } + + var b bytes.Buffer + fmt.Fprintf(&b, "package %s\n\n", pkg.Name()) + for _, imp := range ctx.importsList() { + fmt.Fprintf(&b, "import %q\n", imp) + } + if encoder { + fmt.Fprintln(&b) + b.Write(encSource) + } + if decoder { + fmt.Fprintln(&b) + b.Write(decSource) + } + + source := b.Bytes() + // fmt.Println(string(source)) + return format.Source(source) +} diff --git a/encoding/rlp/rlpgen/gen_test.go b/encoding/rlp/rlpgen/gen_test.go new file mode 100644 index 000000000..241c34b6d --- /dev/null +++ b/encoding/rlp/rlpgen/gen_test.go @@ -0,0 +1,107 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package main + +import ( + "bytes" + "fmt" + "go/ast" + "go/importer" + "go/parser" + "go/token" + "go/types" + "os" + "path/filepath" + "testing" +) + +// Package RLP is loaded only once and reused for all tests. +var ( + testFset = token.NewFileSet() + testImporter = importer.ForCompiler(testFset, "source", nil).(types.ImporterFrom) + testPackageRLP *types.Package +) + +func init() { + cwd, err := os.Getwd() + if err != nil { + panic(err) + } + testPackageRLP, err = testImporter.ImportFrom(pathOfPackageRLP, cwd, 0) + if err != nil { + panic(fmt.Errorf("can't load package RLP: %v", err)) + } +} + +var tests = []string{"uints", "nil", "rawvalue", "optional", "bigint"} + +func TestOutput(t *testing.T) { + for _, test := range tests { + test := test + t.Run(test, func(t *testing.T) { + inputFile := filepath.Join("testdata", test+".in.txt") + outputFile := filepath.Join("testdata", test+".out.txt") + bctx, typ, err := loadTestSource(inputFile, "Test") + if err != nil { + t.Fatal("error loading test source:", err) + } + output, err := bctx.generate(typ, true, true) + if err != nil { + t.Fatal("error in generate:", err) + } + + // Set this environment variable to regenerate the test outputs. + if os.Getenv("WRITE_TEST_FILES") != "" { + os.WriteFile(outputFile, output, 0644) + } + + // Check if output matches. + wantOutput, err := os.ReadFile(outputFile) + if err != nil { + t.Fatal("error loading expected test output:", err) + } + if !bytes.Equal(output, wantOutput) { + t.Fatal("output mismatch:\n", string(output)) + } + }) + } +} + +func loadTestSource(file string, typeName string) (*buildContext, *types.Named, error) { + // Load the test input. + content, err := os.ReadFile(file) + if err != nil { + return nil, nil, err + } + f, err := parser.ParseFile(testFset, file, content, 0) + if err != nil { + return nil, nil, err + } + conf := types.Config{Importer: testImporter} + pkg, err := conf.Check("test", testFset, []*ast.File{f}, nil) + if err != nil { + return nil, nil, err + } + + // Find the test struct. + bctx := newBuildContext(testPackageRLP) + typ, err := lookupStructType(pkg.Scope(), typeName) + if err != nil { + return nil, nil, fmt.Errorf("can't find type %s: %v", typeName, err) + } + return bctx, typ, nil +} diff --git a/encoding/rlp/rlpgen/main.go b/encoding/rlp/rlpgen/main.go new file mode 100644 index 000000000..a447bfe81 --- /dev/null +++ b/encoding/rlp/rlpgen/main.go @@ -0,0 +1,147 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package main + +import ( + "bytes" + "errors" + "flag" + "fmt" + "go/types" + "os" + + "golang.org/x/tools/go/packages" +) + +const pathOfPackageRLP = "github.com/onflow/flow-go-sdk/encoding/rlp" + +func main() { + var ( + pkgdir = flag.String("dir", ".", "input package") + output = flag.String("out", "-", "output file (default is stdout)") + genEncoder = flag.Bool("encoder", true, "generate EncodeRLP?") + genDecoder = flag.Bool("decoder", false, "generate DecodeRLP?") + typename = flag.String("type", "", "type to generate methods for") + ) + flag.Parse() + + cfg := Config{ + Dir: *pkgdir, + Type: *typename, + GenerateEncoder: *genEncoder, + GenerateDecoder: *genDecoder, + } + code, err := cfg.process() + if err != nil { + fatal(err) + } + if *output == "-" { + os.Stdout.Write(code) + } else if err := os.WriteFile(*output, code, 0644); err != nil { + fatal(err) + } +} + +func fatal(args ...interface{}) { + fmt.Fprintln(os.Stderr, args...) + os.Exit(1) +} + +type Config struct { + Dir string // input package directory + Type string + + GenerateEncoder bool + GenerateDecoder bool +} + +// process generates the Go code. +func (cfg *Config) process() (code []byte, err error) { + // Load packages. + pcfg := &packages.Config{ + Mode: packages.NeedName | packages.NeedTypes | packages.NeedImports | packages.NeedDeps, + Dir: cfg.Dir, + BuildFlags: []string{"-tags", "norlpgen"}, + } + ps, err := packages.Load(pcfg, pathOfPackageRLP, ".") + if err != nil { + return nil, err + } + if len(ps) == 0 { + return nil, fmt.Errorf("no Go package found in %s", cfg.Dir) + } + packages.PrintErrors(ps) + + // Find the packages that were loaded. + var ( + pkg *types.Package + packageRLP *types.Package + ) + for _, p := range ps { + if len(p.Errors) > 0 { + return nil, fmt.Errorf("package %s has errors", p.PkgPath) + } + if p.PkgPath == pathOfPackageRLP { + packageRLP = p.Types + } else { + pkg = p.Types + } + } + bctx := newBuildContext(packageRLP) + + // Find the type and generate. + typ, err := lookupStructType(pkg.Scope(), cfg.Type) + if err != nil { + return nil, fmt.Errorf("can't find %s in %s: %v", typ, pkg, err) + } + code, err = bctx.generate(typ, cfg.GenerateEncoder, cfg.GenerateDecoder) + if err != nil { + return nil, err + } + + // Add build comments. + // This is done here to avoid processing these lines with gofmt. + var header bytes.Buffer + fmt.Fprint(&header, "// Code generated by rlpgen. DO NOT EDIT.\n\n") + fmt.Fprint(&header, "//go:build !norlpgen\n") + fmt.Fprint(&header, "// +build !norlpgen\n\n") + return append(header.Bytes(), code...), nil +} + +func lookupStructType(scope *types.Scope, name string) (*types.Named, error) { + typ, err := lookupType(scope, name) + if err != nil { + return nil, err + } + _, ok := typ.Underlying().(*types.Struct) + if !ok { + return nil, errors.New("not a struct type") + } + return typ, nil +} + +func lookupType(scope *types.Scope, name string) (*types.Named, error) { + obj := scope.Lookup(name) + if obj == nil { + return nil, errors.New("no such identifier") + } + typ, ok := obj.(*types.TypeName) + if !ok { + return nil, errors.New("not a type") + } + return typ.Type().(*types.Named), nil +} diff --git a/encoding/rlp/rlpgen/testdata/bigint.in.txt b/encoding/rlp/rlpgen/testdata/bigint.in.txt new file mode 100644 index 000000000..d23d84a28 --- /dev/null +++ b/encoding/rlp/rlpgen/testdata/bigint.in.txt @@ -0,0 +1,10 @@ +// -*- mode: go -*- + +package test + +import "math/big" + +type Test struct { + Int *big.Int + IntNoPtr big.Int +} diff --git a/encoding/rlp/rlpgen/testdata/bigint.out.txt b/encoding/rlp/rlpgen/testdata/bigint.out.txt new file mode 100644 index 000000000..aed7d5f17 --- /dev/null +++ b/encoding/rlp/rlpgen/testdata/bigint.out.txt @@ -0,0 +1,49 @@ +package test + +import "github.com/onflow/flow-go-sdk/encoding/rlp" +import "io" + +func (obj *Test) EncodeRLP(_w io.Writer) error { + w := rlp.NewEncoderBuffer(_w) + _tmp0 := w.List() + if obj.Int == nil { + w.Write(rlp.EmptyString) + } else { + if obj.Int.Sign() == -1 { + return rlp.ErrNegativeBigInt + } + w.WriteBigInt(obj.Int) + } + if obj.IntNoPtr.Sign() == -1 { + return rlp.ErrNegativeBigInt + } + w.WriteBigInt(&obj.IntNoPtr) + w.ListEnd(_tmp0) + return w.Flush() +} + +func (obj *Test) DecodeRLP(dec *rlp.Stream) error { + var _tmp0 Test + { + if _, err := dec.List(); err != nil { + return err + } + // Int: + _tmp1, err := dec.BigInt() + if err != nil { + return err + } + _tmp0.Int = _tmp1 + // IntNoPtr: + _tmp2, err := dec.BigInt() + if err != nil { + return err + } + _tmp0.IntNoPtr = (*_tmp2) + if err := dec.ListEnd(); err != nil { + return err + } + } + *obj = _tmp0 + return nil +} diff --git a/encoding/rlp/rlpgen/testdata/nil.in.txt b/encoding/rlp/rlpgen/testdata/nil.in.txt new file mode 100644 index 000000000..a28ff3448 --- /dev/null +++ b/encoding/rlp/rlpgen/testdata/nil.in.txt @@ -0,0 +1,30 @@ +// -*- mode: go -*- + +package test + +type Aux struct{ + A uint32 +} + +type Test struct{ + Uint8 *byte `rlp:"nil"` + Uint8List *byte `rlp:"nilList"` + + Uint32 *uint32 `rlp:"nil"` + Uint32List *uint32 `rlp:"nilList"` + + Uint64 *uint64 `rlp:"nil"` + Uint64List *uint64 `rlp:"nilList"` + + String *string `rlp:"nil"` + StringList *string `rlp:"nilList"` + + ByteArray *[3]byte `rlp:"nil"` + ByteArrayList *[3]byte `rlp:"nilList"` + + ByteSlice *[]byte `rlp:"nil"` + ByteSliceList *[]byte `rlp:"nilList"` + + Struct *Aux `rlp:"nil"` + StructString *Aux `rlp:"nilString"` +} diff --git a/encoding/rlp/rlpgen/testdata/nil.out.txt b/encoding/rlp/rlpgen/testdata/nil.out.txt new file mode 100644 index 000000000..6ac5f4d08 --- /dev/null +++ b/encoding/rlp/rlpgen/testdata/nil.out.txt @@ -0,0 +1,289 @@ +package test + +import "github.com/onflow/flow-go-sdk/encoding/rlp" +import "io" + +func (obj *Test) EncodeRLP(_w io.Writer) error { + w := rlp.NewEncoderBuffer(_w) + _tmp0 := w.List() + if obj.Uint8 == nil { + w.Write([]byte{0x80}) + } else { + w.WriteUint64(uint64((*obj.Uint8))) + } + if obj.Uint8List == nil { + w.Write([]byte{0xC0}) + } else { + w.WriteUint64(uint64((*obj.Uint8List))) + } + if obj.Uint32 == nil { + w.Write([]byte{0x80}) + } else { + w.WriteUint64(uint64((*obj.Uint32))) + } + if obj.Uint32List == nil { + w.Write([]byte{0xC0}) + } else { + w.WriteUint64(uint64((*obj.Uint32List))) + } + if obj.Uint64 == nil { + w.Write([]byte{0x80}) + } else { + w.WriteUint64((*obj.Uint64)) + } + if obj.Uint64List == nil { + w.Write([]byte{0xC0}) + } else { + w.WriteUint64((*obj.Uint64List)) + } + if obj.String == nil { + w.Write([]byte{0x80}) + } else { + w.WriteString((*obj.String)) + } + if obj.StringList == nil { + w.Write([]byte{0xC0}) + } else { + w.WriteString((*obj.StringList)) + } + if obj.ByteArray == nil { + w.Write([]byte{0x80}) + } else { + w.WriteBytes(obj.ByteArray[:]) + } + if obj.ByteArrayList == nil { + w.Write([]byte{0xC0}) + } else { + w.WriteBytes(obj.ByteArrayList[:]) + } + if obj.ByteSlice == nil { + w.Write([]byte{0x80}) + } else { + w.WriteBytes((*obj.ByteSlice)) + } + if obj.ByteSliceList == nil { + w.Write([]byte{0xC0}) + } else { + w.WriteBytes((*obj.ByteSliceList)) + } + if obj.Struct == nil { + w.Write([]byte{0xC0}) + } else { + _tmp1 := w.List() + w.WriteUint64(uint64(obj.Struct.A)) + w.ListEnd(_tmp1) + } + if obj.StructString == nil { + w.Write([]byte{0x80}) + } else { + _tmp2 := w.List() + w.WriteUint64(uint64(obj.StructString.A)) + w.ListEnd(_tmp2) + } + w.ListEnd(_tmp0) + return w.Flush() +} + +func (obj *Test) DecodeRLP(dec *rlp.Stream) error { + var _tmp0 Test + { + if _, err := dec.List(); err != nil { + return err + } + // Uint8: + var _tmp2 *byte + if _tmp3, _tmp4, err := dec.Kind(); err != nil { + return err + } else if _tmp4 != 0 || _tmp3 != rlp.String { + _tmp1, err := dec.Uint8() + if err != nil { + return err + } + _tmp2 = &_tmp1 + } + _tmp0.Uint8 = _tmp2 + // Uint8List: + var _tmp6 *byte + if _tmp7, _tmp8, err := dec.Kind(); err != nil { + return err + } else if _tmp8 != 0 || _tmp7 != rlp.List { + _tmp5, err := dec.Uint8() + if err != nil { + return err + } + _tmp6 = &_tmp5 + } + _tmp0.Uint8List = _tmp6 + // Uint32: + var _tmp10 *uint32 + if _tmp11, _tmp12, err := dec.Kind(); err != nil { + return err + } else if _tmp12 != 0 || _tmp11 != rlp.String { + _tmp9, err := dec.Uint32() + if err != nil { + return err + } + _tmp10 = &_tmp9 + } + _tmp0.Uint32 = _tmp10 + // Uint32List: + var _tmp14 *uint32 + if _tmp15, _tmp16, err := dec.Kind(); err != nil { + return err + } else if _tmp16 != 0 || _tmp15 != rlp.List { + _tmp13, err := dec.Uint32() + if err != nil { + return err + } + _tmp14 = &_tmp13 + } + _tmp0.Uint32List = _tmp14 + // Uint64: + var _tmp18 *uint64 + if _tmp19, _tmp20, err := dec.Kind(); err != nil { + return err + } else if _tmp20 != 0 || _tmp19 != rlp.String { + _tmp17, err := dec.Uint64() + if err != nil { + return err + } + _tmp18 = &_tmp17 + } + _tmp0.Uint64 = _tmp18 + // Uint64List: + var _tmp22 *uint64 + if _tmp23, _tmp24, err := dec.Kind(); err != nil { + return err + } else if _tmp24 != 0 || _tmp23 != rlp.List { + _tmp21, err := dec.Uint64() + if err != nil { + return err + } + _tmp22 = &_tmp21 + } + _tmp0.Uint64List = _tmp22 + // String: + var _tmp26 *string + if _tmp27, _tmp28, err := dec.Kind(); err != nil { + return err + } else if _tmp28 != 0 || _tmp27 != rlp.String { + _tmp25, err := dec.String() + if err != nil { + return err + } + _tmp26 = &_tmp25 + } + _tmp0.String = _tmp26 + // StringList: + var _tmp30 *string + if _tmp31, _tmp32, err := dec.Kind(); err != nil { + return err + } else if _tmp32 != 0 || _tmp31 != rlp.List { + _tmp29, err := dec.String() + if err != nil { + return err + } + _tmp30 = &_tmp29 + } + _tmp0.StringList = _tmp30 + // ByteArray: + var _tmp34 *[3]byte + if _tmp35, _tmp36, err := dec.Kind(); err != nil { + return err + } else if _tmp36 != 0 || _tmp35 != rlp.String { + var _tmp33 [3]byte + if err := dec.ReadBytes(_tmp33[:]); err != nil { + return err + } + _tmp34 = &_tmp33 + } + _tmp0.ByteArray = _tmp34 + // ByteArrayList: + var _tmp38 *[3]byte + if _tmp39, _tmp40, err := dec.Kind(); err != nil { + return err + } else if _tmp40 != 0 || _tmp39 != rlp.List { + var _tmp37 [3]byte + if err := dec.ReadBytes(_tmp37[:]); err != nil { + return err + } + _tmp38 = &_tmp37 + } + _tmp0.ByteArrayList = _tmp38 + // ByteSlice: + var _tmp42 *[]byte + if _tmp43, _tmp44, err := dec.Kind(); err != nil { + return err + } else if _tmp44 != 0 || _tmp43 != rlp.String { + _tmp41, err := dec.Bytes() + if err != nil { + return err + } + _tmp42 = &_tmp41 + } + _tmp0.ByteSlice = _tmp42 + // ByteSliceList: + var _tmp46 *[]byte + if _tmp47, _tmp48, err := dec.Kind(); err != nil { + return err + } else if _tmp48 != 0 || _tmp47 != rlp.List { + _tmp45, err := dec.Bytes() + if err != nil { + return err + } + _tmp46 = &_tmp45 + } + _tmp0.ByteSliceList = _tmp46 + // Struct: + var _tmp51 *Aux + if _tmp52, _tmp53, err := dec.Kind(); err != nil { + return err + } else if _tmp53 != 0 || _tmp52 != rlp.List { + var _tmp49 Aux + { + if _, err := dec.List(); err != nil { + return err + } + // A: + _tmp50, err := dec.Uint32() + if err != nil { + return err + } + _tmp49.A = _tmp50 + if err := dec.ListEnd(); err != nil { + return err + } + } + _tmp51 = &_tmp49 + } + _tmp0.Struct = _tmp51 + // StructString: + var _tmp56 *Aux + if _tmp57, _tmp58, err := dec.Kind(); err != nil { + return err + } else if _tmp58 != 0 || _tmp57 != rlp.String { + var _tmp54 Aux + { + if _, err := dec.List(); err != nil { + return err + } + // A: + _tmp55, err := dec.Uint32() + if err != nil { + return err + } + _tmp54.A = _tmp55 + if err := dec.ListEnd(); err != nil { + return err + } + } + _tmp56 = &_tmp54 + } + _tmp0.StructString = _tmp56 + if err := dec.ListEnd(); err != nil { + return err + } + } + *obj = _tmp0 + return nil +} diff --git a/encoding/rlp/rlpgen/testdata/optional.in.txt b/encoding/rlp/rlpgen/testdata/optional.in.txt new file mode 100644 index 000000000..f1ac9f789 --- /dev/null +++ b/encoding/rlp/rlpgen/testdata/optional.in.txt @@ -0,0 +1,17 @@ +// -*- mode: go -*- + +package test + +type Aux struct { + A uint64 +} + +type Test struct { + Uint64 uint64 `rlp:"optional"` + Pointer *uint64 `rlp:"optional"` + String string `rlp:"optional"` + Slice []uint64 `rlp:"optional"` + Array [3]byte `rlp:"optional"` + NamedStruct Aux `rlp:"optional"` + AnonStruct struct{ A string } `rlp:"optional"` +} diff --git a/encoding/rlp/rlpgen/testdata/optional.out.txt b/encoding/rlp/rlpgen/testdata/optional.out.txt new file mode 100644 index 000000000..7dde68583 --- /dev/null +++ b/encoding/rlp/rlpgen/testdata/optional.out.txt @@ -0,0 +1,153 @@ +package test + +import "github.com/onflow/flow-go-sdk/encoding/rlp" +import "io" + +func (obj *Test) EncodeRLP(_w io.Writer) error { + w := rlp.NewEncoderBuffer(_w) + _tmp0 := w.List() + _tmp1 := obj.Uint64 != 0 + _tmp2 := obj.Pointer != nil + _tmp3 := obj.String != "" + _tmp4 := len(obj.Slice) > 0 + _tmp5 := obj.Array != ([3]byte{}) + _tmp6 := obj.NamedStruct != (Aux{}) + _tmp7 := obj.AnonStruct != (struct{ A string }{}) + if _tmp1 || _tmp2 || _tmp3 || _tmp4 || _tmp5 || _tmp6 || _tmp7 { + w.WriteUint64(obj.Uint64) + } + if _tmp2 || _tmp3 || _tmp4 || _tmp5 || _tmp6 || _tmp7 { + if obj.Pointer == nil { + w.Write([]byte{0x80}) + } else { + w.WriteUint64((*obj.Pointer)) + } + } + if _tmp3 || _tmp4 || _tmp5 || _tmp6 || _tmp7 { + w.WriteString(obj.String) + } + if _tmp4 || _tmp5 || _tmp6 || _tmp7 { + _tmp8 := w.List() + for _, _tmp9 := range obj.Slice { + w.WriteUint64(_tmp9) + } + w.ListEnd(_tmp8) + } + if _tmp5 || _tmp6 || _tmp7 { + w.WriteBytes(obj.Array[:]) + } + if _tmp6 || _tmp7 { + _tmp10 := w.List() + w.WriteUint64(obj.NamedStruct.A) + w.ListEnd(_tmp10) + } + if _tmp7 { + _tmp11 := w.List() + w.WriteString(obj.AnonStruct.A) + w.ListEnd(_tmp11) + } + w.ListEnd(_tmp0) + return w.Flush() +} + +func (obj *Test) DecodeRLP(dec *rlp.Stream) error { + var _tmp0 Test + { + if _, err := dec.List(); err != nil { + return err + } + // Uint64: + if dec.MoreDataInList() { + _tmp1, err := dec.Uint64() + if err != nil { + return err + } + _tmp0.Uint64 = _tmp1 + // Pointer: + if dec.MoreDataInList() { + _tmp2, err := dec.Uint64() + if err != nil { + return err + } + _tmp0.Pointer = &_tmp2 + // String: + if dec.MoreDataInList() { + _tmp3, err := dec.String() + if err != nil { + return err + } + _tmp0.String = _tmp3 + // Slice: + if dec.MoreDataInList() { + var _tmp4 []uint64 + if _, err := dec.List(); err != nil { + return err + } + for dec.MoreDataInList() { + _tmp5, err := dec.Uint64() + if err != nil { + return err + } + _tmp4 = append(_tmp4, _tmp5) + } + if err := dec.ListEnd(); err != nil { + return err + } + _tmp0.Slice = _tmp4 + // Array: + if dec.MoreDataInList() { + var _tmp6 [3]byte + if err := dec.ReadBytes(_tmp6[:]); err != nil { + return err + } + _tmp0.Array = _tmp6 + // NamedStruct: + if dec.MoreDataInList() { + var _tmp7 Aux + { + if _, err := dec.List(); err != nil { + return err + } + // A: + _tmp8, err := dec.Uint64() + if err != nil { + return err + } + _tmp7.A = _tmp8 + if err := dec.ListEnd(); err != nil { + return err + } + } + _tmp0.NamedStruct = _tmp7 + // AnonStruct: + if dec.MoreDataInList() { + var _tmp9 struct{ A string } + { + if _, err := dec.List(); err != nil { + return err + } + // A: + _tmp10, err := dec.String() + if err != nil { + return err + } + _tmp9.A = _tmp10 + if err := dec.ListEnd(); err != nil { + return err + } + } + _tmp0.AnonStruct = _tmp9 + } + } + } + } + } + } + } + if err := dec.ListEnd(); err != nil { + return err + } + } + *obj = _tmp0 + return nil +} diff --git a/encoding/rlp/rlpgen/testdata/rawvalue.in.txt b/encoding/rlp/rlpgen/testdata/rawvalue.in.txt new file mode 100644 index 000000000..0e69aa286 --- /dev/null +++ b/encoding/rlp/rlpgen/testdata/rawvalue.in.txt @@ -0,0 +1,11 @@ +// -*- mode: go -*- + +package test + +import "github.com/onflow/flow-go-sdk/encoding/rlp" + +type Test struct { + RawValue rlp.RawValue + PointerToRawValue *rlp.RawValue + SliceOfRawValue []rlp.RawValue +} diff --git a/encoding/rlp/rlpgen/testdata/rawvalue.out.txt b/encoding/rlp/rlpgen/testdata/rawvalue.out.txt new file mode 100644 index 000000000..7eadf6d78 --- /dev/null +++ b/encoding/rlp/rlpgen/testdata/rawvalue.out.txt @@ -0,0 +1,64 @@ +package test + +import "github.com/onflow/flow-go-sdk/encoding/rlp" +import "io" + +func (obj *Test) EncodeRLP(_w io.Writer) error { + w := rlp.NewEncoderBuffer(_w) + _tmp0 := w.List() + w.Write(obj.RawValue) + if obj.PointerToRawValue == nil { + w.Write([]byte{0x80}) + } else { + w.Write((*obj.PointerToRawValue)) + } + _tmp1 := w.List() + for _, _tmp2 := range obj.SliceOfRawValue { + w.Write(_tmp2) + } + w.ListEnd(_tmp1) + w.ListEnd(_tmp0) + return w.Flush() +} + +func (obj *Test) DecodeRLP(dec *rlp.Stream) error { + var _tmp0 Test + { + if _, err := dec.List(); err != nil { + return err + } + // RawValue: + _tmp1, err := dec.Raw() + if err != nil { + return err + } + _tmp0.RawValue = _tmp1 + // PointerToRawValue: + _tmp2, err := dec.Raw() + if err != nil { + return err + } + _tmp0.PointerToRawValue = &_tmp2 + // SliceOfRawValue: + var _tmp3 []rlp.RawValue + if _, err := dec.List(); err != nil { + return err + } + for dec.MoreDataInList() { + _tmp4, err := dec.Raw() + if err != nil { + return err + } + _tmp3 = append(_tmp3, _tmp4) + } + if err := dec.ListEnd(); err != nil { + return err + } + _tmp0.SliceOfRawValue = _tmp3 + if err := dec.ListEnd(); err != nil { + return err + } + } + *obj = _tmp0 + return nil +} diff --git a/encoding/rlp/rlpgen/testdata/uints.in.txt b/encoding/rlp/rlpgen/testdata/uints.in.txt new file mode 100644 index 000000000..8095da997 --- /dev/null +++ b/encoding/rlp/rlpgen/testdata/uints.in.txt @@ -0,0 +1,10 @@ +// -*- mode: go -*- + +package test + +type Test struct{ + A uint8 + B uint16 + C uint32 + D uint64 +} diff --git a/encoding/rlp/rlpgen/testdata/uints.out.txt b/encoding/rlp/rlpgen/testdata/uints.out.txt new file mode 100644 index 000000000..b93e91415 --- /dev/null +++ b/encoding/rlp/rlpgen/testdata/uints.out.txt @@ -0,0 +1,53 @@ +package test + +import "github.com/onflow/flow-go-sdk/encoding/rlp" +import "io" + +func (obj *Test) EncodeRLP(_w io.Writer) error { + w := rlp.NewEncoderBuffer(_w) + _tmp0 := w.List() + w.WriteUint64(uint64(obj.A)) + w.WriteUint64(uint64(obj.B)) + w.WriteUint64(uint64(obj.C)) + w.WriteUint64(obj.D) + w.ListEnd(_tmp0) + return w.Flush() +} + +func (obj *Test) DecodeRLP(dec *rlp.Stream) error { + var _tmp0 Test + { + if _, err := dec.List(); err != nil { + return err + } + // A: + _tmp1, err := dec.Uint8() + if err != nil { + return err + } + _tmp0.A = _tmp1 + // B: + _tmp2, err := dec.Uint16() + if err != nil { + return err + } + _tmp0.B = _tmp2 + // C: + _tmp3, err := dec.Uint32() + if err != nil { + return err + } + _tmp0.C = _tmp3 + // D: + _tmp4, err := dec.Uint64() + if err != nil { + return err + } + _tmp0.D = _tmp4 + if err := dec.ListEnd(); err != nil { + return err + } + } + *obj = _tmp0 + return nil +} diff --git a/encoding/rlp/rlpgen/types.go b/encoding/rlp/rlpgen/types.go new file mode 100644 index 000000000..19694262e --- /dev/null +++ b/encoding/rlp/rlpgen/types.go @@ -0,0 +1,114 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package main + +import ( + "fmt" + "go/types" + "reflect" +) + +// typeReflectKind gives the reflect.Kind that represents typ. +func typeReflectKind(typ types.Type) reflect.Kind { + switch typ := typ.(type) { + case *types.Basic: + k := typ.Kind() + if k >= types.Bool && k <= types.Complex128 { + // value order matches for Bool..Complex128 + return reflect.Bool + reflect.Kind(k-types.Bool) + } + if k == types.String { + return reflect.String + } + if k == types.UnsafePointer { + return reflect.UnsafePointer + } + panic(fmt.Errorf("unhandled BasicKind %v", k)) + case *types.Array: + return reflect.Array + case *types.Chan: + return reflect.Chan + case *types.Interface: + return reflect.Interface + case *types.Map: + return reflect.Map + case *types.Pointer: + return reflect.Ptr + case *types.Signature: + return reflect.Func + case *types.Slice: + return reflect.Slice + case *types.Struct: + return reflect.Struct + default: + panic(fmt.Errorf("unhandled type %T", typ)) + } +} + +// nonZeroCheck returns the expression that checks whether 'v' is a non-zero value of type 'vtyp'. +func nonZeroCheck(v string, vtyp types.Type, qualify types.Qualifier) string { + // Resolve type name. + typ := resolveUnderlying(vtyp) + switch typ := typ.(type) { + case *types.Basic: + k := typ.Kind() + switch { + case k == types.Bool: + return v + case k >= types.Uint && k <= types.Complex128: + return fmt.Sprintf("%s != 0", v) + case k == types.String: + return fmt.Sprintf(`%s != ""`, v) + default: + panic(fmt.Errorf("unhandled BasicKind %v", k)) + } + case *types.Array, *types.Struct: + return fmt.Sprintf("%s != (%s{})", v, types.TypeString(vtyp, qualify)) + case *types.Interface, *types.Pointer, *types.Signature: + return fmt.Sprintf("%s != nil", v) + case *types.Slice, *types.Map: + return fmt.Sprintf("len(%s) > 0", v) + default: + panic(fmt.Errorf("unhandled type %T", typ)) + } +} + +// isBigInt checks whether 'typ' is "math/big".Int. +func isBigInt(typ types.Type) bool { + named, ok := typ.(*types.Named) + if !ok { + return false + } + name := named.Obj() + return name.Pkg().Path() == "math/big" && name.Name() == "Int" +} + +// isByte checks whether the underlying type of 'typ' is uint8. +func isByte(typ types.Type) bool { + basic, ok := resolveUnderlying(typ).(*types.Basic) + return ok && basic.Kind() == types.Uint8 +} + +func resolveUnderlying(typ types.Type) types.Type { + for { + t := typ.Underlying() + if t == typ { + return t + } + typ = t + } +} diff --git a/encoding/rlp/safe.go b/encoding/rlp/safe.go new file mode 100644 index 000000000..3c910337b --- /dev/null +++ b/encoding/rlp/safe.go @@ -0,0 +1,27 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +//go:build nacl || js || !cgo +// +build nacl js !cgo + +package rlp + +import "reflect" + +// byteArrayBytes returns a slice of the byte array v. +func byteArrayBytes(v reflect.Value, length int) []byte { + return v.Slice(0, length).Bytes() +} diff --git a/encoding/rlp/typecache.go b/encoding/rlp/typecache.go new file mode 100644 index 000000000..1e749062d --- /dev/null +++ b/encoding/rlp/typecache.go @@ -0,0 +1,240 @@ +// Copyright 2014 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rlp + +import ( + "fmt" + "reflect" + "sync" + "sync/atomic" + + "github.com/onflow/flow-go-sdk/encoding/rlp/internal/rlpstruct" +) + +// typeinfo is an entry in the type cache. +type typeinfo struct { + decoder decoder + decoderErr error // error from makeDecoder + writer writer + writerErr error // error from makeWriter +} + +// typekey is the key of a type in typeCache. It includes the struct tags because +// they might generate a different decoder. +type typekey struct { + reflect.Type + rlpstruct.Tags +} + +type decoder func(*Stream, reflect.Value) error + +type writer func(reflect.Value, *encBuffer) error + +var theTC = newTypeCache() + +type typeCache struct { + cur atomic.Value + + // This lock synchronizes writers. + mu sync.Mutex + next map[typekey]*typeinfo +} + +func newTypeCache() *typeCache { + c := new(typeCache) + c.cur.Store(make(map[typekey]*typeinfo)) + return c +} + +func cachedDecoder(typ reflect.Type) (decoder, error) { + info := theTC.info(typ) + return info.decoder, info.decoderErr +} + +func cachedWriter(typ reflect.Type) (writer, error) { + info := theTC.info(typ) + return info.writer, info.writerErr +} + +func (c *typeCache) info(typ reflect.Type) *typeinfo { + key := typekey{Type: typ} + if info := c.cur.Load().(map[typekey]*typeinfo)[key]; info != nil { + return info + } + + // Not in the cache, need to generate info for this type. + return c.generate(typ, rlpstruct.Tags{}) +} + +func (c *typeCache) generate(typ reflect.Type, tags rlpstruct.Tags) *typeinfo { + c.mu.Lock() + defer c.mu.Unlock() + + cur := c.cur.Load().(map[typekey]*typeinfo) + if info := cur[typekey{typ, tags}]; info != nil { + return info + } + + // Copy cur to next. + c.next = make(map[typekey]*typeinfo, len(cur)+1) + for k, v := range cur { + c.next[k] = v + } + + // Generate. + info := c.infoWhileGenerating(typ, tags) + + // next -> cur + c.cur.Store(c.next) + c.next = nil + return info +} + +func (c *typeCache) infoWhileGenerating(typ reflect.Type, tags rlpstruct.Tags) *typeinfo { + key := typekey{typ, tags} + if info := c.next[key]; info != nil { + return info + } + // Put a dummy value into the cache before generating. + // If the generator tries to lookup itself, it will get + // the dummy value and won't call itself recursively. + info := new(typeinfo) + c.next[key] = info + info.generate(typ, tags) + return info +} + +type field struct { + index int + info *typeinfo + optional bool +} + +// structFields resolves the typeinfo of all public fields in a struct type. +func structFields(typ reflect.Type) (fields []field, err error) { + // Convert fields to rlpstruct.Field. + var allStructFields []rlpstruct.Field + for i := 0; i < typ.NumField(); i++ { + rf := typ.Field(i) + allStructFields = append(allStructFields, rlpstruct.Field{ + Name: rf.Name, + Index: i, + Exported: rf.PkgPath == "", + Tag: string(rf.Tag), + Type: *rtypeToStructType(rf.Type, nil), + }) + } + + // Filter/validate fields. + structFields, structTags, err := rlpstruct.ProcessFields(allStructFields) + if err != nil { + if tagErr, ok := err.(rlpstruct.TagError); ok { + tagErr.StructType = typ.String() + return nil, tagErr + } + return nil, err + } + + // Resolve typeinfo. + for i, sf := range structFields { + typ := typ.Field(sf.Index).Type + tags := structTags[i] + info := theTC.infoWhileGenerating(typ, tags) + fields = append(fields, field{sf.Index, info, tags.Optional}) + } + return fields, nil +} + +// firstOptionalField returns the index of the first field with "optional" tag. +func firstOptionalField(fields []field) int { + for i, f := range fields { + if f.optional { + return i + } + } + return len(fields) +} + +type structFieldError struct { + typ reflect.Type + field int + err error +} + +func (e structFieldError) Error() string { + return fmt.Sprintf("%v (struct field %v.%s)", e.err, e.typ, e.typ.Field(e.field).Name) +} + +func (i *typeinfo) generate(typ reflect.Type, tags rlpstruct.Tags) { + i.decoder, i.decoderErr = makeDecoder(typ, tags) + i.writer, i.writerErr = makeWriter(typ, tags) +} + +// rtypeToStructType converts typ to rlpstruct.Type. +func rtypeToStructType(typ reflect.Type, rec map[reflect.Type]*rlpstruct.Type) *rlpstruct.Type { + k := typ.Kind() + if k == reflect.Invalid { + panic("invalid kind") + } + + if prev := rec[typ]; prev != nil { + return prev // short-circuit for recursive types + } + if rec == nil { + rec = make(map[reflect.Type]*rlpstruct.Type) + } + + t := &rlpstruct.Type{ + Name: typ.String(), + Kind: k, + IsEncoder: typ.Implements(encoderInterface), + IsDecoder: typ.Implements(decoderInterface), + } + rec[typ] = t + if k == reflect.Array || k == reflect.Slice || k == reflect.Ptr { + t.Elem = rtypeToStructType(typ.Elem(), rec) + } + return t +} + +// typeNilKind gives the RLP value kind for nil pointers to 'typ'. +func typeNilKind(typ reflect.Type, tags rlpstruct.Tags) Kind { + styp := rtypeToStructType(typ, nil) + + var nk rlpstruct.NilKind + if tags.NilOK { + nk = tags.NilKind + } else { + nk = styp.DefaultNilValue() + } + switch nk { + case rlpstruct.NilKindString: + return String + case rlpstruct.NilKindList: + return List + default: + panic("invalid nil kind value") + } +} + +func isUint(k reflect.Kind) bool { + return k >= reflect.Uint && k <= reflect.Uintptr +} + +func isByte(typ reflect.Type) bool { + return typ.Kind() == reflect.Uint8 && !typ.Implements(encoderInterface) +} diff --git a/encoding/rlp/unsafe.go b/encoding/rlp/unsafe.go new file mode 100644 index 000000000..2152ba35f --- /dev/null +++ b/encoding/rlp/unsafe.go @@ -0,0 +1,35 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +//go:build !nacl && !js && cgo +// +build !nacl,!js,cgo + +package rlp + +import ( + "reflect" + "unsafe" +) + +// byteArrayBytes returns a slice of the byte array v. +func byteArrayBytes(v reflect.Value, length int) []byte { + var s []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&s)) + hdr.Data = v.UnsafeAddr() + hdr.Cap = length + hdr.Len = length + return s +} diff --git a/flow.go b/flow.go index b42bc262d..4fe274a4a 100644 --- a/flow.go +++ b/flow.go @@ -22,9 +22,8 @@ import ( "encoding/hex" "sync" - "github.com/ethereum/go-ethereum/rlp" - "github.com/onflow/flow-go-sdk/crypto" + "github.com/onflow/flow-go-sdk/encoding/rlp" ) // An Identifier is a 32-byte unique identifier for an entity. diff --git a/go.mod b/go.mod index d244a1c0c..de2a3fbaf 100644 --- a/go.mod +++ b/go.mod @@ -4,13 +4,14 @@ go 1.16 require ( cloud.google.com/go/kms v1.4.0 - github.com/ethereum/go-ethereum v1.9.13 github.com/onflow/cadence v0.23.4 github.com/onflow/flow-go/crypto v0.24.3 github.com/onflow/flow/protobuf/go/flow v0.2.5 github.com/onflow/sdks v0.4.4 github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.7.1 + golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect + golang.org/x/tools v0.1.5 // indirect google.golang.org/api v0.70.0 google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf google.golang.org/grpc v1.44.0 diff --git a/transaction.go b/transaction.go index 67bddceed..6fc8ff661 100644 --- a/transaction.go +++ b/transaction.go @@ -24,9 +24,9 @@ import ( "fmt" "sort" - "github.com/ethereum/go-ethereum/rlp" "github.com/onflow/cadence" jsoncdc "github.com/onflow/cadence/encoding/json" + "github.com/onflow/flow-go-sdk/encoding/rlp" "github.com/onflow/flow-go-sdk/crypto" )