Add support for arbitrary size integers

ultrajson · Jun 15, 2022 · ec07551 · ec07551
1 parent d40cf50
commit ec07551
Show file tree

Hide file tree

Showing 4 changed files with 36 additions and 1 deletion.
diff --git a/lib/ultrajson.h b/lib/ultrajson.h
@@ -332,6 +332,7 @@ typedef struct __JSONObjectDecoder
   JSOBJ (*newInt)(void *prv, JSINT32 value);
   JSOBJ (*newLong)(void *prv, JSINT64 value);
   JSOBJ (*newUnsignedLong)(void *prv, JSUINT64 value);
+  JSOBJ (*newIntegerFromString)(void *prv, char *value, size_t length);
   JSOBJ (*newDouble)(void *prv, double value);
   void (*releaseObject)(void *prv, JSOBJ obj);
   JSPFN_MALLOC malloc;

diff --git a/lib/ultrajsondec.c b/lib/ultrajsondec.c
@@ -173,7 +173,10 @@ static FASTCALL_ATTR JSOBJ FASTCALL_MSVC decode_numeric (struct DecoderState *ds
       {
         if (hasError)
         {
-          return SetError(ds, -1, intNeg == 1 ? "Value is too big" : "Value is too small");
+          char *strStart = ds->start;
+          ds->lastType = JT_INT;
+          ds->start = offset;
+          return ds->dec->newIntegerFromString(ds->prv, strStart, offset - strStart);
         }
         goto BREAK_INT_LOOP;
         break;

diff --git a/python/JSONtoObj.c b/python/JSONtoObj.c
@@ -119,6 +119,15 @@ static JSOBJ Object_newUnsignedLong(void *prv, JSUINT64 value)
   return PyLong_FromUnsignedLongLong (value);
 }
 
+static JSOBJ Object_newIntegerFromString(void *prv, char *value, size_t length)
+{
+  // PyLong_FromString requires a NUL-terminated string in CPython, contrary to the documentation: https://github.com/python/cpython/issues/59200
+  char *buf = PyObject_Malloc(length + 1);
+  memcpy(buf, value, length);
+  buf[length] = '\0';
+  return PyLong_FromString(buf, NULL, 10);
+}
+
 static JSOBJ Object_newDouble(void *prv, double value)
 {
   return PyFloat_FromDouble(value);
@@ -152,6 +161,7 @@ PyObject* JSONToObj(PyObject* self, PyObject *args, PyObject *kwargs)
     Object_newInteger,
     Object_newLong,
     Object_newUnsignedLong,
+    Object_newIntegerFromString,
     Object_newDouble,
     Object_releaseObject,
     PyObject_Malloc,

diff --git a/python/objToJSON.c b/python/objToJSON.c
@@ -100,6 +100,17 @@ static void *PyLongToUINT64(JSOBJ _obj, JSONTypeContext *tc, void *outValue, siz
   return NULL;
 }
 
+static void *PyLongToINTSTR(JSOBJ _obj, JSONTypeContext *tc, void *outValue, size_t *_outLen)
+{
+  PyObject *obj = PyNumber_ToBase(_obj, 10);
+  if (!obj)
+  {
+    return NULL;
+  }
+  *_outLen = PyUnicode_GET_LENGTH(obj);
+  return PyUnicode_1BYTE_DATA(obj);
+}
+
 static void *PyFloatToDOUBLE(JSOBJ _obj, JSONTypeContext *tc, void *outValue, size_t *_outLen)
 {
   PyObject *obj = (PyObject *) _obj;
@@ -508,6 +519,16 @@ static void Object_beginTypeContext (JSOBJ _obj, JSONTypeContext *tc, JSONObject
       exc = PyErr_Occurred();
     }
 
+    if (exc && PyErr_ExceptionMatches(PyExc_OverflowError))
+    {
+      PyErr_Clear();
+      pc->PyTypeToJSON = PyLongToINTSTR;
+      tc->type = JT_RAW;
+      // Overwritten by PyLong_* due to the union, which would lead to a DECREF in endTypeContext.
+      GET_TC(tc)->rawJSONValue = NULL;
+      return;
+    }
+
     if (exc)
     {
       PRINTMARK();