Skip to content

feat: i/o for DocVec #1562

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 26 commits into from
Jun 28, 2023
Merged
Changes from 1 commit
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
d43057b
feat: json and dict for docvec
JohannesMessner May 22, 2023
c45bfca
test: add tests
JohannesMessner May 22, 2023
564d144
test: add docvec to dict test
JohannesMessner May 22, 2023
76f9c8e
feat: to from dataframe for docvec
JohannesMessner May 22, 2023
73a1ac7
test: dataframe docvec tests
JohannesMessner May 22, 2023
f83fb4f
feat: to from csv for docvec
JohannesMessner May 22, 2023
ca8dc12
test: test csv with docvec
JohannesMessner May 22, 2023
2b52b1e
Merge branch 'main' into feat-docvec-io
JohannesMessner Jun 14, 2023
b115637
feat: pickle serialization for docvec
JohannesMessner Jun 14, 2023
bd86985
feat: protbuf array serialization for docvec
JohannesMessner Jun 14, 2023
c280ff2
test: test base64 deser for docvec
JohannesMessner Jun 14, 2023
ad881cf
test: test save and load for docvec
JohannesMessner Jun 14, 2023
4b1b533
feat: docvec json column wise
JohannesMessner Jun 19, 2023
60e651e
Merge branch 'main' into feat-docvec-io
JohannesMessner Jun 19, 2023
f9c97ec
Merge branch 'main' into feat-docvec-io
JohannesMessner Jun 20, 2023
0603fc5
test: add test for docvec json
JohannesMessner Jun 20, 2023
c6ace8e
test: add tensor type arg
JohannesMessner Jun 20, 2023
51719b2
fix: mypy stuff
JohannesMessner Jun 26, 2023
ad5f5bd
fix: raising of error when needed
JohannesMessner Jun 26, 2023
200dbac
fix: more exception raising
JohannesMessner Jun 26, 2023
8d1f446
fix: mypy
JohannesMessner Jun 26, 2023
6815720
refactor: don't expose to/from csv for docvec
JohannesMessner Jun 26, 2023
6b5ddc7
test: adjust tests
JohannesMessner Jun 26, 2023
587c20a
docs: add documentation for docvec io
JohannesMessner Jun 27, 2023
663f17d
Merge branch 'main' into feat-docvec-io
JohannesMessner Jun 27, 2023
7d035fb
Merge branch 'main' into feat-docvec-io
JohannesMessner Jun 28, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
test: dataframe docvec tests
Signed-off-by: Johannes Messner <messnerjo@gmail.com>
  • Loading branch information
JohannesMessner committed May 22, 2023
commit 73a1ac7417b1192562d240a4413300e78953f9a7
25 changes: 18 additions & 7 deletions tests/units/array/test_array_from_to_pandas.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import pandas as pd
import pytest

from docarray import BaseDoc, DocList
from docarray import BaseDoc, DocList, DocVec
from docarray.documents import ImageDoc


Expand All @@ -19,7 +19,8 @@ class MyDocNested(MyDoc):
return MyDocNested


def test_to_from_pandas_df(nested_doc_cls):
@pytest.mark.parametrize('doc_vec', [False, True])
def test_to_from_pandas_df(nested_doc_cls, doc_vec):
da = DocList[nested_doc_cls](
[
nested_doc_cls(
Expand All @@ -30,6 +31,8 @@ def test_to_from_pandas_df(nested_doc_cls):
nested_doc_cls(text='hello world', image=ImageDoc()),
]
)
if doc_vec:
da = da.to_doc_vec()
df = da.to_dataframe()
assert isinstance(df, pd.DataFrame)
assert len(df) == 2
Expand All @@ -47,7 +50,12 @@ def test_to_from_pandas_df(nested_doc_cls):
]
).all()

da_from_df = DocList[nested_doc_cls].from_dataframe(df)
if doc_vec:
da_from_df = DocVec[nested_doc_cls].from_dataframe(df)
assert isinstance(da_from_df, DocVec)
else:
da_from_df = DocList[nested_doc_cls].from_dataframe(df)
assert isinstance(da_from_df, DocList)
for doc1, doc2 in zip(da, da_from_df):
assert doc1 == doc2

Expand All @@ -71,26 +79,29 @@ class Outer(BaseDoc):
return doc


def test_from_pandas_without_schema_raise_exception():
@pytest.mark.parametrize('array_cls', [DocList, DocVec])
def test_from_pandas_without_schema_raise_exception(array_cls):
with pytest.raises(TypeError, match='no document schema defined'):
df = pd.DataFrame(
columns=['title', 'count'], data=[['title 0', 0], ['title 1', 1]]
)
DocList.from_dataframe(df=df)
array_cls.from_dataframe(df=df)


def test_from_pandas_with_wrong_schema_raise_exception(nested_doc):
@pytest.mark.parametrize('array_cls', [DocList, DocVec])
def test_from_pandas_with_wrong_schema_raise_exception(nested_doc, array_cls):
with pytest.raises(ValueError, match='Column names do not match the schema'):
df = pd.DataFrame(
columns=['title', 'count'], data=[['title 0', 0], ['title 1', 1]]
)
DocList[nested_doc.__class__].from_dataframe(df=df)
array_cls[nested_doc.__class__].from_dataframe(df=df)


def test_doc_list_error():
class Book(BaseDoc):
title: str

# not testing DocVec bc it already fails here (as it should!)
docs = DocList([Book(title='hello'), Book(title='world')])
with pytest.raises(TypeError):
docs.to_dataframe()
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy