Skip to content

Commit dc9172a

Browse files
lostmsulostmsu
committed
implements buffer interface for .NET arrays of primitive types
fixes losttech/Gradient#27
1 parent 1e32d8c commit dc9172a

File tree

10 files changed

+291
-55
lines changed

10 files changed

+291
-55
lines changed

‎.github/workflows/main.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ jobs:
4141
- name: Install dependencies
4242
run: |
4343
pip install --upgrade -r requirements.txt
44+
pip install numpy # for tests
4445
4546
- name: Build and Install
4647
run: |

‎CHANGELOG.md

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,10 @@ This document follows the conventions laid out in [Keep a CHANGELOG][].
1515
- Ability to implement delegates with `ref` and `out` parameters in Python, by returning the modified parameter values in a tuple. ([#1355][i1355])
1616
- `PyType` - a wrapper for Python type objects, that also permits creating new heap types from `TypeSpec`
1717
- Improved exception handling:
18-
- exceptions can now be converted with codecs
19-
- `InnerException` and `__cause__` are propagated properly
18+
- exceptions can now be converted with codecs
19+
- `InnerException` and `__cause__` are propagated properly
20+
- .NET arrays implement Python buffer protocol
21+
2022

2123
### Changed
2224
- Drop support for Python 2, 3.4, and 3.5

‎src/embed_tests/NumPyTests.cs

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using NUnit.Framework;
4+
using Python.Runtime;
5+
using Python.Runtime.Codecs;
6+
7+
namespace Python.EmbeddingTest
8+
{
9+
public class NumPyTests
10+
{
11+
[OneTimeSetUp]
12+
public void SetUp()
13+
{
14+
PythonEngine.Initialize();
15+
TupleCodec<ValueTuple>.Register();
16+
}
17+
18+
[OneTimeTearDown]
19+
public void Dispose()
20+
{
21+
PythonEngine.Shutdown();
22+
}
23+
24+
[Test]
25+
public void TestReadme()
26+
{
27+
dynamic np;
28+
try
29+
{
30+
np = Py.Import("numpy");
31+
}
32+
catch (PythonException)
33+
{
34+
Assert.Inconclusive("Numpy or dependency not installed");
35+
return;
36+
}
37+
38+
Assert.AreEqual("1.0", np.cos(np.pi * 2).ToString());
39+
40+
dynamic sin = np.sin;
41+
StringAssert.StartsWith("-0.95892", sin(5).ToString());
42+
43+
double c = np.cos(5) + sin(5);
44+
Assert.AreEqual(-0.675262, c, 0.01);
45+
46+
dynamic a = np.array(new List<float> { 1, 2, 3 });
47+
Assert.AreEqual("float64", a.dtype.ToString());
48+
49+
dynamic b = np.array(new List<float> { 6, 5, 4 }, Py.kw("dtype", np.int32));
50+
Assert.AreEqual("int32", b.dtype.ToString());
51+
52+
Assert.AreEqual("[ 6. 10. 12.]", (a * b).ToString().Replace(" ", " "));
53+
}
54+
55+
[Test]
56+
public void MultidimensionalNumPyArray()
57+
{
58+
PyObject np;
59+
try {
60+
np = Py.Import("numpy");
61+
} catch (PythonException) {
62+
Assert.Inconclusive("Numpy or dependency not installed");
63+
return;
64+
}
65+
66+
var array = new[,] { { 1, 2 }, { 3, 4 } };
67+
var ndarray = np.InvokeMethod("asarray", array.ToPython());
68+
Assert.AreEqual((2,2), ndarray.GetAttr("shape").As<(int,int)>());
69+
Assert.AreEqual(1, ndarray[(0, 0).ToPython()].InvokeMethod("__int__").As<int>());
70+
Assert.AreEqual(array[1, 0], ndarray[(1, 0).ToPython()].InvokeMethod("__int__").As<int>());
71+
}
72+
73+
[Test]
74+
public void Int64Array()
75+
{
76+
PyObject np;
77+
try
78+
{
79+
np = Py.Import("numpy");
80+
}
81+
catch (PythonException)
82+
{
83+
Assert.Inconclusive("Numpy or dependency not installed");
84+
return;
85+
}
86+
87+
var array = new long[,] { { 1, 2 }, { 3, 4 } };
88+
var ndarray = np.InvokeMethod("asarray", array.ToPython());
89+
Assert.AreEqual((2, 2), ndarray.GetAttr("shape").As<(int, int)>());
90+
Assert.AreEqual(1, ndarray[(0, 0).ToPython()].InvokeMethod("__int__").As<long>());
91+
Assert.AreEqual(array[1, 0], ndarray[(1, 0).ToPython()].InvokeMethod("__int__").As<long>());
92+
}
93+
}
94+
}

‎src/embed_tests/TestExample.cs

Lines changed: 0 additions & 53 deletions
This file was deleted.

‎src/embed_tests/TestPyBuffer.cs

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
1+
using System;
12
using System.Text;
23
using NUnit.Framework;
34
using Python.Runtime;
5+
using Python.Runtime.Codecs;
46

57
namespace Python.EmbeddingTest {
68
class TestPyBuffer
@@ -9,6 +11,7 @@ class TestPyBuffer
911
public void SetUp()
1012
{
1113
PythonEngine.Initialize();
14+
TupleCodec<ValueTuple>.Register();
1215
}
1316

1417
[OneTimeTearDown]
@@ -64,5 +67,15 @@ public void TestBufferRead()
6467
}
6568
}
6669
}
70+
71+
[Test]
72+
public void ArrayHasBuffer()
73+
{
74+
var array = new[,] {{1, 2}, {3,4}};
75+
var memoryView = PythonEngine.Eval("memoryview");
76+
var mem = memoryView.Invoke(array.ToPython());
77+
Assert.AreEqual(1, mem[(0, 0).ToPython()].As<int>());
78+
Assert.AreEqual(array[1,0], mem[(1, 0).ToPython()].As<int>());
79+
}
6780
}
6881
}

‎src/runtime/arrayobject.cs

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
using System;
22
using System.Collections;
3+
using System.Collections.Generic;
4+
using System.Runtime.InteropServices;
35

46
namespace Python.Runtime
57
{
@@ -366,5 +368,166 @@ public static int sq_contains(IntPtr ob, IntPtr v)
366368

367369
return 0;
368370
}
371+
372+
#region Buffer protocol
373+
static int GetBuffer(BorrowedReference obj, out Py_buffer buffer, PyBUF flags)
374+
{
375+
buffer = default;
376+
377+
if (flags == PyBUF.SIMPLE)
378+
{
379+
Exceptions.SetError(Exceptions.BufferError, "SIMPLE not implemented");
380+
return -1;
381+
}
382+
if ((flags & PyBUF.F_CONTIGUOUS) == PyBUF.F_CONTIGUOUS)
383+
{
384+
Exceptions.SetError(Exceptions.BufferError, "only C-contiguous supported");
385+
return -1;
386+
}
387+
var self = (Array)((CLRObject)GetManagedObject(obj)).inst;
388+
Type itemType = self.GetType().GetElementType();
389+
390+
bool formatRequested = (flags & PyBUF.FORMATS) != 0;
391+
string format = GetFormat(itemType);
392+
if (formatRequested && format is null)
393+
{
394+
Exceptions.SetError(Exceptions.BufferError, "unsupported element type: " + itemType.Name);
395+
return -1;
396+
}
397+
GCHandle gcHandle;
398+
try
399+
{
400+
gcHandle = GCHandle.Alloc(self, GCHandleType.Pinned);
401+
} catch (ArgumentException ex)
402+
{
403+
Exceptions.SetError(Exceptions.BufferError, ex.Message);
404+
return -1;
405+
}
406+
407+
int itemSize = Marshal.SizeOf(itemType);
408+
IntPtr[] shape = GetShape(self);
409+
IntPtr[] strides = GetStrides(shape, itemSize);
410+
buffer = new Py_buffer
411+
{
412+
buf = gcHandle.AddrOfPinnedObject(),
413+
obj = Runtime.SelfIncRef(obj.DangerousGetAddress()),
414+
len = (IntPtr)(self.LongLength*itemSize),
415+
itemsize = (IntPtr)itemSize,
416+
_readonly = false,
417+
ndim = self.Rank,
418+
format = format,
419+
shape = ToUnmanaged(shape),
420+
strides = (flags & PyBUF.STRIDES) == PyBUF.STRIDES ? ToUnmanaged(strides) : IntPtr.Zero,
421+
suboffsets = IntPtr.Zero,
422+
_internal = (IntPtr)gcHandle,
423+
};
424+
425+
return 0;
426+
}
427+
static void ReleaseBuffer(BorrowedReference obj, ref Py_buffer buffer)
428+
{
429+
if (buffer._internal == IntPtr.Zero) return;
430+
431+
UnmanagedFree(ref buffer.shape);
432+
UnmanagedFree(ref buffer.strides);
433+
UnmanagedFree(ref buffer.suboffsets);
434+
435+
var gcHandle = (GCHandle)buffer._internal;
436+
gcHandle.Free();
437+
buffer._internal = IntPtr.Zero;
438+
}
439+
440+
static IntPtr[] GetStrides(IntPtr[] shape, long itemSize)
441+
{
442+
var result = new IntPtr[shape.Length];
443+
result[shape.Length - 1] = new IntPtr(itemSize);
444+
for (int dim = shape.Length - 2; dim >= 0; dim--)
445+
{
446+
itemSize *= shape[dim + 1].ToInt64();
447+
result[dim] = new IntPtr(itemSize);
448+
}
449+
return result;
450+
}
451+
static IntPtr[] GetShape(Array array)
452+
{
453+
var result = new IntPtr[array.Rank];
454+
for (int i = 0; i < result.Length; i++)
455+
result[i] = (IntPtr)array.GetLongLength(i);
456+
return result;
457+
}
458+
459+
static void UnmanagedFree(ref IntPtr address)
460+
{
461+
if (address == IntPtr.Zero) return;
462+
463+
Marshal.FreeHGlobal(address);
464+
address = IntPtr.Zero;
465+
}
466+
static unsafe IntPtr ToUnmanaged<T>(T[] array) where T : unmanaged
467+
{
468+
IntPtr result = Marshal.AllocHGlobal(checked(Marshal.SizeOf(typeof(T)) * array.Length));
469+
fixed (T* ptr = array)
470+
{
471+
var @out = (T*)result;
472+
for (int i = 0; i < array.Length; i++)
473+
@out[i] = ptr[i];
474+
}
475+
return result;
476+
}
477+
478+
static readonly Dictionary<Type, string> ItemFormats = new Dictionary<Type, string>
479+
{
480+
[typeof(byte)] = "B",
481+
[typeof(sbyte)] = "b",
482+
483+
[typeof(bool)] = "?",
484+
485+
[typeof(short)] = "h",
486+
[typeof(ushort)] = "H",
487+
// see https://github.com/pybind/pybind11/issues/1908#issuecomment-658358767
488+
[typeof(int)] = "i",
489+
[typeof(uint)] = "I",
490+
[typeof(long)] = "q",
491+
[typeof(ulong)] = "Q",
492+
493+
[typeof(IntPtr)] = "n",
494+
[typeof(UIntPtr)] = "N",
495+
496+
// TODO: half = "e"
497+
[typeof(float)] = "f",
498+
[typeof(double)] = "d",
499+
};
500+
501+
static string GetFormat(Type elementType)
502+
=> ItemFormats.TryGetValue(elementType, out string result) ? result : null;
503+
504+
static readonly GetBufferProc getBufferProc = GetBuffer;
505+
static readonly ReleaseBufferProc releaseBufferProc = ReleaseBuffer;
506+
static readonly IntPtr BufferProcsAddress = AllocateBufferProcs();
507+
static IntPtr AllocateBufferProcs()
508+
{
509+
var procs = new PyBufferProcs
510+
{
511+
Get = Marshal.GetFunctionPointerForDelegate(getBufferProc),
512+
Release = Marshal.GetFunctionPointerForDelegate(releaseBufferProc),
513+
};
514+
IntPtr result = Marshal.AllocHGlobal(Marshal.SizeOf(typeof(PyBufferProcs)));
515+
Marshal.StructureToPtr(procs, result, fDeleteOld: false);
516+
return result;
517+
}
518+
#endregion
519+
520+
/// <summary>
521+
/// <see cref="TypeManager.InitializeSlots(IntPtr, Type, SlotsHolder)"/>
522+
/// </summary>
523+
public static void InitializeSlots(IntPtr type, ISet<string> initialized, SlotsHolder slotsHolder)
524+
{
525+
if (initialized.Add(nameof(TypeOffset.tp_as_buffer)))
526+
{
527+
// TODO: only for unmanaged arrays
528+
int offset = TypeOffset.GetSlotOffset(nameof(TypeOffset.tp_as_buffer));
529+
Marshal.WriteIntPtr(type, offset, BufferProcsAddress);
530+
}
531+
}
369532
}
370533
}

‎src/runtime/bufferinterface.cs

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,4 +103,15 @@ public enum PyBUF
103103
/// </summary>
104104
FULL_RO = (INDIRECT | FORMATS),
105105
}
106+
107+
internal struct PyBufferProcs
108+
{
109+
public IntPtr Get;
110+
public IntPtr Release;
111+
}
112+
113+
[UnmanagedFunctionPointer(CallingConvention.Cdecl)]
114+
delegate int GetBufferProc(BorrowedReference obj, out Py_buffer buffer, PyBUF flags);
115+
[UnmanagedFunctionPointer(CallingConvention.Cdecl)]
116+
delegate void ReleaseBufferProc(BorrowedReference obj, ref Py_buffer buffer);
106117
}

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy