Merge pull request #8004 from wizrds/fix/dataframe_json

Revert DataFrame serialization in Producer mode (>pandas 2.0)
This commit is contained in:
Matthias 2023-05-22 19:58:05 +02:00 committed by GitHub
commit 5b29e710dc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

View File

@ -8,7 +8,6 @@ from pathlib import Path
from typing import Any, Dict, Iterator, List, Mapping, Optional, TextIO, Union
from urllib.parse import urlparse
import orjson
import pandas as pd
import rapidjson
@ -249,17 +248,7 @@ def dataframe_to_json(dataframe: pd.DataFrame) -> str:
:param dataframe: A pandas DataFrame
:returns: A JSON string of the pandas DataFrame
"""
# https://github.com/pandas-dev/pandas/issues/24889
# https://github.com/pandas-dev/pandas/issues/40443
# We need to convert to a dict to avoid mem leak
def default(z):
if isinstance(z, pd.Timestamp):
return z.timestamp() * 1e3
if z is pd.NaT:
return 'NaT'
raise TypeError
return str(orjson.dumps(dataframe.to_dict(orient='split'), default=default), 'utf-8')
return dataframe.to_json(orient='split')
def json_to_dataframe(data: str) -> pd.DataFrame: