Convert JSON lines to JSON array using jq
I use one liner of node call:
$ cat input.ndjson | node -e 'const rl = readline.createInterface({ input: process.stdin }); !async function () { let idx = 0; for await (const line of rl) { process.stdout.write((++idx === 1 ? "[" : "\n,") + JSON.stringify(JSON.parse(line))); } process.stdout.write("]"); }()' | tee output.json | jq 'length'
16814
Explanation:
$ cat input.ndjson | # pipe the ndjson input, can be a file or any stream
node -e '
const rl = readline.createInterface({ input: process.stdin });
!async function () { // to be able to use await, this function has to be async, and then `!` is to be able call it right away;
let idx = 0;
for await (const line of rl) {
process.stdout.write((++idx === 1 ? "[" : "\n,") + JSON.stringify(JSON.parse(line)));
}
process.stdout.write("]");
}()
' | tee output.json # save the output json file
| jq 'length' # call jq to calculate a length, also validate it, to make sure it's a valid single json file, this optional
the readline of Node is a good one to read a stream by lines, and it's asyncIterable, can use (await ... of) to get each line, then call JSON.parse to make sure validate each line is a valid json, and then JSON.stringify to convert back to a minified JSON, on each line;
for sure you can change to a local const arr = []; and push each line's obj into t, and once JSON.stringify at the end, to get the most minified JSON,
but I like to get this almost minified format, that each line's obj is minified, but still keep the outer array line by line, so that I can easily get its total count by a wc -l count its lines
[{...minified line1obj with no spaces...}
,{...minified line2obj}
,{...minified line3obj}
,...
,{...minified lineNobj}]
It looks like inputs suffers from the same issue as slurp. I don't know how to accomplish this with jq on huge files, but sed can do it:
sed '1s/^/[/; $!s/$/,/; $s/$/]/' in.json > out.json
Output:
[{"Header":{"AssetClass":"Commodities","InstrumentType":"Forward","UseCase":"Forward","Level":"InstRefDataReporting"},"Attributes":{"NotionalCurrency":"EUR","ExpiryDate":"2018-01-01","ReturnorPayoutTrigger":"Contract for Difference (CFD)","DeliveryType":"CASH","BaseProduct":"AGRI","TransactionType":"FUTR","FinalPriceType":"ARGM","ReferenceRate":"10PPM ULTRA LOW SULPHUR DIESEL-CARGOES CIF NWE/BASIS ARA-PLATTS EUROPEAN","SubProduct":"GROS","AdditionalSubProduct":"FWHT"},"ISIN":{"ISIN":"EZX27M86B860","Status":"New"},"TemplateVersion":1,"Derived":{"CommodityDerivativeIndicator":"TRUE","UnderlyingAssetType":"Agriculture","IssuerorOperatoroftheTradingVenueIdentifier":"NA","PriceMultiplier":1,"FullName":"Commodities Forward AGRI GROS FWHT EUR 20180101","ShortName":"NA/Fwd AGRI FWHT EUR 20180101","ClassificationType":"JTAXCC"}},
{"Header":{"AssetClass":"Commodities","InstrumentType":"Swap","UseCase":"Basis_Swap","Level":"InstRefDataReporting"},"Attributes":{"NotionalCurrency":"SOS","ExpiryDate":"2208-12-10","ReturnorPayoutTrigger":"Total Return","DeliveryType":"OPTL","TransactionType":"ORIT","FinalPriceType":"IHSM","ReferenceRate":"NATURAL GAS-MONTHLY INDEX S. TEXAS (TETCO)-GAS DAILY PRICE GUIDE","OtherReferenceRate":"NATURAL GAS-MONTHLY INDEX W. LOUISIANA (TETCO)-GAS DAILY PRICE GUIDE","BaseProduct":"OTHR","OtherBaseProduct":"OTHR","SubProduct":"","AdditionalSubProduct":"","OtherSubProduct":"","OtherAdditionalSubProduct":""},"ISIN":{"ISIN":"EZBBH1XR9GV6","Status":"New"},"TemplateVersion":1,"Derived":{"CommodityDerivativeIndicator":"TRUE","UnderlyingAssetType":"Multi Commodity","IssuerorOperatoroftheTradingVenueIdentifier":"NA","PriceMultiplier":1,"FullName":"Commodities Swap Basis_Swap OTHR OTHR SOS 22081210","ShortName":"NA/Swap OTHR SOS 22081210","ClassificationType":"STQTXE"}},
{"Header":{"AssetClass":"Commodities","InstrumentType":"Swap","UseCase":"Multi_Exotic_Swap","Level":"InstRefDataReporting"},"Attributes":{"NotionalCurrency":"LRD","ExpiryDate":"2200-01-31","ReturnorPayoutTrigger":"Contract for Difference (CFD)","DeliveryType":"CASH","TransactionType":"TAPO","FinalPriceType":"EXOF","UnderlyingInstrumentIndex":["BCOMF6","BCOMNG3"]},"ISIN":{"ISIN":"EZ286HJVY4Q2","Status":"New"},"TemplateVersion":1,"Derived":{"CommodityDerivativeIndicator":"TRUE","IssuerorOperatoroftheTradingVenueIdentifier":"NA","PriceMultiplier":1,"UnderlyingAssetType":"Multi Commodity","BaseProduct":"MCEX","SubProduct":"","AdditionalSubProduct":"","FullName":"Commodities Multi_Exotic_Swap MCEX LRD 22000131","ShortName":"NA/Swap MCEX LRD 22000131","ClassificationType":"STQCXC"}},
{"Header":{"AssetClass":"Commodities","InstrumentType":"Option","UseCase":"Option","Level":"InstRefDataReporting"},"Attributes":{"NotionalCurrency":"TND","ExpiryDate":"2209-10-18","OptionType":"OPTL","OptionExerciseStyle":"AMER","ValuationMethodorTrigger":"Asian","DeliveryType":"CASH","TransactionType":"OTHR","FinalPriceType":"IHSM","ReferenceRate":"NATURAL GAS-NGPL (NICOR, NIPSCO, PGLC CITYGATE), NBPL-NICOR-ICE/10X MONTHLY","BaseProduct":"OTHR","SubProduct":"","AdditionalSubProduct":""},"ISIN":{"ISIN":"EZ2TK5CWL9Y4","Status":"New"},"TemplateVersion":1,"Derived":{"CommodityDerivativeIndicator":"TRUE","UnderlyingAssetType":"Other","IssuerorOperatoroftheTradingVenueIdentifier":"NA","PriceMultiplier":1,"FullName":"Commodities Option OTHR TND 22091018","ShortName":"NA/O OTHR OPTL TND 22091018","ClassificationType":"HTMHAC"}},
{"Header":{"AssetClass":"Commodities","InstrumentType":"Option","UseCase":"Multi_Exotic_Option","Level":"InstRefDataReporting"},"Attributes":{"NotionalCurrency":"SOS","ExpiryDate":"2209-10-18","UnderlyingInstrumentIndex":["BCOMSI2","BCOMPR3T"],"OptionType":"CALL","OptionExerciseStyle":"AMER","ValuationMethodorTrigger":"Other Path Dependent","DeliveryType":"CASH","TransactionType":"ORIT","FinalPriceType":"BLTC"},"ISIN":{"ISIN":"EZ82L36B6225","Status":"New"},"TemplateVersion":1,"Derived":{"CommodityDerivativeIndicator":"TRUE","IssuerorOperatoroftheTradingVenueIdentifier":"NA","PriceMultiplier":1,"UnderlyingAssetType":"Multi Commodity","BaseProduct":"MCEX","SubProduct":"","AdditionalSubProduct":"","FullName":"Commodities Multi_Exotic_Option MCEX SOS 22091018","ShortName":"NA/O MCEX Call SOS 22091018","ClassificationType":"HTQBPC"}}]
Explanation
The sed script consists of three separate substitutions. Here they are on separate lines:
1 s/^/[/ # Insert a left bracket at the beginning of the first line
$! s/$/,/ # On all but the last line append a comma
$ s/$/]/ # Append a right bracket to the last line
Maybe awk is easier on the eyes:
awk 'BEGIN{print "["}
length(last)>0 {print last ","} {last=$0}
END {print last, "]"}'
And for the record, here's a slurp-free solution using a version of jq with input:
jq -nr '"[", try (input|tojson, repeat(",\n\(input|tojson)")), "]"'
Maybe, I am late but this is what you are looking for!
jq -s '.' in.json > out.json