Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ujson.dumps strips out dictionary value #589

Open
diman82 opened this issue Apr 12, 2023 · 3 comments
Open

ujson.dumps strips out dictionary value #589

diman82 opened this issue Apr 12, 2023 · 3 comments

Comments

@diman82
Copy link

diman82 commented Apr 12, 2023

What did you do?

Run ujson.dumps(obj, ensure_ascii=False)

What did you expect to happen?

get json str with all keys

What actually happened?

got partial json str

  • OS: Multi (Windows/Ubuntu)
  • Python: 3.8.4
  • UltraJSON: 5.7.0

Please include code that reproduces the issue.
Original item:

{
    "api_version": "2022-11-15",
    "created": 1680095224,
    "data": {
        "object": {
            "application": null,
            "application_fee_percent": null,
            "automatic_tax": {
                "enabled": true
            },
            "billing_cycle_anchor": 1680095219,
            "billing_thresholds": null,
            "cancel_at": null,
            "cancel_at_period_end": false,
            "canceled_at": null,
            "cancellation_details": {
                "comment": null,
                "feedback": null,
                "reason": null
            },
            "collection_method": "charge_automatically",
            "created": 1680095219,
            "currency": "usd",
            "current_period_end": 1711717619,
            "current_period_start": 1680095219,
            "customer": "cus_NcD8kB8Nmcccez",
            "days_until_due": null,
            "default_payment_method": null,
            "default_source": null,
            "default_tax_rates": [
            ],
            "description": null,
            "discount": null,
            "ended_at": null,
            "id": "sub_1MqyiODuyXTtmLcmhMaPCaFL",
            "items": {
                "data": [
                    {
                        "billing_thresholds": null,
                        "created": 1680095219,
                        "id": "si_NcD8quYc8PzZF5",
                        "metadata": {
                        },
                        "object": "subscription_item",
                        "plan": {
                            "active": true,
                            "aggregate_usage": null,
                            "amount": 349900,
                            "amount_decimal": "349900",
                            "billing_scheme": "per_unit",
                            "created": 1679847073,
                            "currency": "usd",
                            "id": "price_1MpwA1DuyXTtmLcmQld9D6U9",
                            "interval": "year",
                            "interval_count": 1,
                            "livemode": false,
                            "metadata": {
                            },
                            "nickname": "Annual",
                            "object": "plan",
                            "product": "mp_316",
                            "tiers_mode": null,
                            "transform_usage": null,
                            "trial_period_days": null,
                            "usage_type": "licensed"
                        },
                        "price": {
                            "active": true,
                            "billing_scheme": "per_unit",
                            "created": 1679847073,
                            "currency": "usd",
                            "custom_unit_amount": null,
                            "id": "price_1MpwA1DuyXTtmLcmQld9D6U9",
                            "livemode": false,
                            "lookup_key": "mp_316_annual",
                            "metadata": {
                            },
                            "nickname": "Annual",
                            "object": "price",
                            "product": "mp_316",
                            "recurring": {
                                "aggregate_usage": null,
                                "interval": "year",
                                "interval_count": 1,
                                "trial_period_days": null,
                                "usage_type": "licensed"
                            },
                            "tax_behavior": "exclusive",
                            "tiers_mode": null,
                            "transform_quantity": null,
                            "type": "recurring",
                            "unit_amount": 349900,
                            "unit_amount_decimal": "349900"
                        },
                        "quantity": 1,
                        "subscription": "sub_1MqyiODuyXTtmLcmhMaPCaFL",
                        "tax_rates": [
                        ]
                    }
                ],
                "has_more": false,
                "object": "list",
                "total_count": 1,
                "url": "/v1/subscription_items?subscription=sub_1MqyiODuyXTtmLcmhMaPCaFL"
            },
            "latest_invoice": "in_1MqyiODuyXTtmLcmLPFlgwXa",
            "livemode": false,
            "metadata": {
                "author_revenue_share": "0.75",
                "eligible_to_mbg": "1_month",
                "platform_name": "regular",
                "product_id": "mp_316"
            },
            "next_pending_invoice_item_invoice": null,
            "object": "subscription",
            "on_behalf_of": null,
            "pause_collection": null,
            "payment_settings": {
                "payment_method_options": null,
                "payment_method_types": null,
                "save_default_payment_method": "off"
            },
            "pending_invoice_item_interval": null,
            "pending_setup_intent": null,
            "pending_update": null,
            "plan": {
                "active": true,
                "aggregate_usage": null,
                "amount": 349900,
                "amount_decimal": "349900",
                "billing_scheme": "per_unit",
                "created": 1679847073,
                "currency": "usd",
                "id": "price_1MpwA1DuyXTtmLcmQld9D6U9",
                "interval": "year",
                "interval_count": 1,
                "livemode": false,
                "metadata": {
                },
                "nickname": "Annual",
                "object": "plan",
                "product": "mp_316",
                "tiers_mode": null,
                "transform_usage": null,
                "trial_period_days": null,
                "usage_type": "licensed"
            },
            "quantity": 1,
            "schedule": null,
            "start_date": 1680095219,
            "status": "active",
            "test_clock": "clock_1MqyiNDuyXTtmLcm6lnDNrmj",
            "transfer_data": null,
            "trial_end": null,
            "trial_settings": {
                "end_behavior": {
                    "missing_payment_method": "create_invoice"
                }
            },
            "trial_start": null
        },
        "previous_attributes": {
            "status": "incomplete"
        }
    },
    "id": "evt_1MqyiTDuyXTtmLcmyL43ybID",
    "livemode": false,
    "object": "event",
    "pending_webhooks": 0,
    "request": {
        "id": "req_jZVh8Z4ueyGXSz",
        "idempotency_key": "c7357e8c-6061-44de-b5c4-99438e1509d8"
    },
    "type": "customer.subscription.updated"
}

After ujson.dumps(obj, ensure_ascii=False):

{
    "api_version": "2022-11-15",
    "created": 1680095224,
    "data": {
        "object": {
            "application": null,
            "application_fee_percent": null,
            "automatic_tax": {
                "enabled": true
            },
            "billing_cycle_anchor": 1680095219,
            "billing_thresholds": null,
            "cancel_at": null,
            "cancel_at_period_end": false,
            "canceled_at": null,
            "cancellation_details": {
                "comment": null,
                "feedback": null,
                "reason": null
            },
            "collection_method": "charge_automatically",
            "created": 1680095219,
            "currency": "usd",
            "current_period_end": 1711717619,
            "current_period_start": 1680095219,
            "customer": "cus_NcD8kB8Nmcccez",
            "days_until_due": null,
            "default_payment_method": null,
            "default_source": null,
            "default_tax_rates": [
            ],
            "description": null,
            "discount": null,
            "ended_at": null,
            "id": "sub_1MqyiODuyXTtmLcmhMaPCaFL",
            "items": {
            },
            "latest_invoice": "in_1MqyiODuyXTtmLcmLPFlgwXa",
            "livemode": false,
            "metadata": {
                "author_revenue_share": "0.75",
                "eligible_to_mbg": "1_month",
                "platform_name": "regular",
                "product_id": "mp_316"
            },
            "next_pending_invoice_item_invoice": null,
            "object": "subscription",
            "on_behalf_of": null,
            "pause_collection": null,
            "payment_settings": {
                "payment_method_options": null,
                "payment_method_types": null,
                "save_default_payment_method": "off"
            },
            "pending_invoice_item_interval": null,
            "pending_setup_intent": null,
            "pending_update": null,
            "plan": {
                "active": true,
                "aggregate_usage": null,
                "amount": 349900,
                "amount_decimal": "349900",
                "billing_scheme": "per_unit",
                "created": 1679847073,
                "currency": "usd",
                "id": "price_1MpwA1DuyXTtmLcmQld9D6U9",
                "interval": "year",
                "interval_count": 1,
                "livemode": false,
                "metadata": {
                },
                "nickname": "Annual",
                "object": "plan",
                "product": "mp_316",
                "tiers_mode": null,
                "transform_usage": null,
                "trial_period_days": null,
                "usage_type": "licensed"
            },
            "quantity": 1,
            "schedule": null,
            "start_date": 1680095219,
            "status": "active",
            "test_clock": "clock_1MqyiNDuyXTtmLcm6lnDNrmj",
            "transfer_data": null,
            "trial_end": null,
            "trial_settings": {
                "end_behavior": {
                    "missing_payment_method": "create_invoice"
                }
            },
            "trial_start": null
        },
        "previous_attributes": {
            "status": "incomplete"
        }
    },
    "id": "evt_1MqyiTDuyXTtmLcmyL43ybID",
    "livemode": false,
    "object": "event",
    "pending_webhooks": 0,
    "request": {
        "id": "req_jZVh8Z4ueyGXSz",
        "idempotency_key": "c7357e8c-6061-44de-b5c4-99438e1509d8"
    },
    "type": "customer.subscription.updated"
}

res: 'items' key is missing data
p.s. running json.dumps (with same obj and params) yields correct result json str

code goes here
@bwoodsend
Copy link
Collaborator

I can't see any errors:

>>> import json, ujson
>>> obj = json.loads('{"api_version": "2022-11-15", "created": 1680095224, "data": {"object": {"application": null, "application_fee_percent": null, "automatic_tax": {"enabled": true}, "billing_cycle_anchor": 1680095219, "billing_thresholds": null, "cancel_at": null, "cancel_at_period_end": false, "canceled_at": null, "cancellation_details": {"comment": null, "feedback": null, "reason": null}, "collection_method": "charge_automatically", "created": 1680095219, "currency": "usd", "current_period_end": 1711717619, "current_period_start": 1680095219, "customer": "cus_NcD8kB8Nmcccez", "days_until_due": null, "default_payment_method": null, "default_source": null, "default_tax_rates": [], "description" : null, "discount": null, "ended_at": null, "id": "sub_1MqyiODuyXTtmLcmhMaPCaFL", "items": {"data": [{"billing_thresholds": null, "created": 1680095219, "id": "si_NcD8quYc8PzZF5", "metadata": {}, "object": "subscription_item", "plan": {"active": true, "aggregate_usage": null, "amount": 349900, "amount_decimal": "349900", "billing_scheme": "per_unit", "created": 1679847073, "currency": "usd", "id": "price_1MpwA1DuyXTtmLcmQld9D6U9", "interval": "year", "interval_count": 1, "livemode": false, "metadata": {}, "nickname": "Annual", "object": "plan", "product": "mp_316", "tiers_mode": null, "transform_usage": null, "trial_period_days": null, "usage_type": "licensed"}, "price": {"active": true, "billing_scheme": "per_unit", "created": 1679847073, "currency": "usd", "custom_unit_amount": null, "id": "price_1MpwA1DuyXTtmLcmQld9D6U9", "livemode": false, "lookup_key": "mp_316_annual", "metadata": {}, "nickname": "Annual", "object": "price", "product": "mp_316", "recurring": {"aggregate_usage": null, "interval": "year", "interval_count": 1, "trial_period_days": null, "usage_type": "licensed"}, "tax_behavior": "exclusive", "tiers_mode": null, "transform_quantity": null, "type": "recurring", "unit_amount": 349900, "unit_amount_decimal": "349900"}, "quantity": 1, "subscription": "sub_1MqyiODuyXTtmLcmhMaPCaFL", "tax_rates": []}], "has_more": false, "object": "list", "total_count": 1, "url": "/v1/subscription_items?subscription=sub_1MqyiODuyXTtmLcmhMaPCaFL"}, "latest_invoice": "in_1MqyiODuyXTtmLcmLPFlgwXa", "livemode": false, "metadata": {"author_revenue_share": "0.75", "eligible_to_mbg": "1_month", "platform_name": "regular", "product_id": "mp_316"}, "next_pending_invoice_item_invoice": null, "object": "subscription", "on_behalf_of": null, "pause_collection": null, "payment_settings": {"payment_method_options": null, "payment_method_types": null, "save_default_payment_method": "off"}, "pending_invoice_item_interval": null, "pending_setup_intent": null, "pending_update": null, "plan": {"active": true, "aggregate_usage": null, "amount": 349900, "amount_decimal": "349900", "billing_scheme": "per_unit", "created": 1679847073, "currency": "usd", "id": "price_1MpwA1DuyXTtmLcmQld9D6U9", "interval": "year", "interval_count": 1, "livemode": false, "metadata": {}, "nickname": "Annual", "object": "plan", "product": "mp_316", "tiers_mode": null, "transform_usage": null, "trial_period_days": null, "usage_type": "licensed"}, "quantity": 1, "schedule": null, "start_date": 1680095219, "status": "active", "test_clock": "clock_1MqyiNDuyXTtmLcm6lnDNrmj", "transfer_data": null, "trial_end": null, "trial_settings": {"end_behavior": {"missing_payment_method": "create_invoice"}}, "trial_start": null}, "previous_attributes": {"status": "incomplete"}}, "id": "evt_1MqyiTDuyXTtmLcmyL43ybID", "livemode": false, "object": "event", "pending_webhooks": 0, "request": {"id": "req_jZVh8Z4ueyGXSz", "idempotency_key": "c7357e8c-6061-44de-b5c4-99438e1509d8"}, "type": "customer.subscription.updated"}')

>>> ujson.dumps(obj, ensure_ascii=False)
'{"api_version":"2022-11-15","created":1680095224,"data":{"object":{"application":null,"application_fee_percent":null,"automatic_tax":{"enabled":true},"billing_cycle_anchor":1680095219,"billing_thresholds":null,"cancel_at":null,"cancel_at_period_end":false,"canceled_at":null,"cancellation_details":{"comment":null,"feedback":null,"reason":null},"collection_method":"charge_automatically","created":1680095219,"currency":"usd","current_period_end":1711717619,"current_period_start":1680095219,"customer":"cus_NcD8kB8Nmcccez","days_until_due":null,"default_payment_method":null,"default_source":null,"default_tax_rates":[],"description":null,"discount":null,"ended_at":null,"id":"sub_1MqyiODuyXTtmLcmhMaPCaFL","items":{"data":[{"billing_thresholds":null,"created":1680095219,"id":"si_NcD8quYc8PzZF5","metadata":{},"object":"subscription_item","plan":{"active":true,"aggregate_usage":null,"amount":349900,"amount_decimal":"349900","billing_scheme":"per_unit","created":1679847073,"currency":"usd","id":"price_1MpwA1DuyXTtmLcmQld9D6U9","interval":"year","interval_count":1,"livemode":false,"metadata":{},"nickname":"Annual","object":"plan","product":"mp_316","tiers_mode":null,"transform_usage":null,"trial_period_days":null,"usage_type":"licensed"},"price":{"active":true,"billing_scheme":"per_unit","created":1679847073,"currency":"usd","custom_unit_amount":null,"id":"price_1MpwA1DuyXTtmLcmQld9D6U9","livemode":false,"lookup_key":"mp_316_annual","metadata":{},"nickname":"Annual","object":"price","product":"mp_316","recurring":{"aggregate_usage":null,"interval":"year","interval_count":1,"trial_period_days":null,"usage_type":"licensed"},"tax_behavior":"exclusive","tiers_mode":null,"transform_quantity":null,"type":"recurring","unit_amount":349900,"unit_amount_decimal":"349900"},"quantity":1,"subscription":"sub_1MqyiODuyXTtmLcmhMaPCaFL","tax_rates":[]}],"has_more":false,"object":"list","total_count":1,"url":"\\/v1\\/subscription_items?subscription=sub_1MqyiODuyXTtmLcmhMaPCaFL"},"latest_invoice":"in_1MqyiODuyXTtmLcmLPFlgwXa","livemode":false,"metadata":{"author_revenue_share":"0.75","eligible_to_mbg":"1_month","platform_name":"regular","product_id":"mp_316"},"next_pending_invoice_item_invoice":null,"object":"subscription","on_behalf_of":null,"pause_collection":null,"payment_settings":{"payment_method_options":null,"payment_method_types":null,"save_default_payment_method":"off"},"pending_invoice_item_interval":null,"pending_setup_intent":null,"pending_update":null,"plan":{"active":true,"aggregate_usage":null,"amount":349900,"amount_decimal":"349900","billing_scheme":"per_unit","created":1679847073,"currency":"usd","id":"price_1MpwA1DuyXTtmLcmQld9D6U9","interval":"year","interval_count":1,"livemode":false,"metadata":{},"nickname":"Annual","object":"plan","product":"mp_316","tiers_mode":null,"transform_usage":null,"trial_period_days":null,"usage_type":"licensed"},"quantity":1,"schedule":null,"start_date":1680095219,"status":"active","test_clock":"clock_1MqyiNDuyXTtmLcm6lnDNrmj","transfer_data":null,"trial_end":null,"trial_settings":{"end_behavior":{"missing_payment_method":"create_invoice"}},"trial_start":null},"previous_attributes":{"status":"incomplete"}},"id":"evt_1MqyiTDuyXTtmLcmyL43ybID","livemode":false,"object":"event","pending_webhooks":0,"request":{"id":"req_jZVh8Z4ueyGXSz","idempotency_key":"c7357e8c-6061-44de-b5c4-99438e1509d8"},"type":"customer.subscription.updated"}'

>>> json.loads(ujson.dumps(obj, ensure_ascii=False)) == obj
True

Are you sure you're doing what you think you're doing?

@diman82
Copy link
Author

diman82 commented Apr 13, 2023

@bwoodsend Not sure why changing to json solved this issue, but here is my code fragment:

with NamedTemporaryFile("w") as tmp:
    # added workaround to convert to NDJSON format
    for record_value in event_type_values.values():
        tmp.write(json.dumps(record_value, ensure_ascii=False) + os.linesep)

So, if you change to ujson, 'items' array gets skipped (you won't find it in result stream/file)

@bwoodsend
Copy link
Collaborator

bwoodsend commented Apr 13, 2023

Please give a complete self contained example. We're not going to get anywhere if I have to guess what event_type_values contains.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Labels
None yet
Projects
None yet
Development

No branches or pull requests

2 participants