Skip to content

Type tweaks #67

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mwiebe merged 6 commits into from
Apr 22, 2014
Merged

Type tweaks #67

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
c5cc362
Update to reflect types changes in libdynd
mwiebe Apr 20, 2014
cc7aad2
Update for ctuple type id
mwiebe Apr 20, 2014
089493b
Use ndt::type::tcast<>() as added to libdynd
mwiebe Apr 20, 2014
e414491
Update to reflect new fixed[N] type
mwiebe Apr 21, 2014
1ca7b43
Update the code/tests to pass with the libdynd changes
mwiebe Apr 21, 2014
509eb8d
Fixes for issues exposed by the blaze tests
mwiebe Apr 21, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions buildscripts/jenkins-build
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ fi
export EXTRA="-DDYND_BUILD_TESTS=OFF ${EXTRA}"

# Create a fresh makefile with cmake, and do the build/install
cd build
pushd build
cmake ${EXTRA} \
-DCMAKE_INSTALL_PREFIX=${PYENV_PREFIX} \
-DPYTHON_EXECUTABLE=${PYTHON_EXECUTABLE} \
Expand All @@ -100,9 +100,9 @@ if [ '${PYDYND_VERSION}' == '' ]; then
exit 1
fi
export PYDYND_VERSION=${PYDYND_VERSION//-/_}
popd

# Put the conda package by itself in the directory pkgs
cd ..
rm -rf pkgs
mkdir pkgs
cd pkgs
Expand Down
38 changes: 20 additions & 18 deletions dynd/_lowlevel/type_id.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@
'POINTER', 'BYTES', 'FIXEDBYTES',
'CHAR', 'STRING', 'FIXEDSTRING',
'CATEGORICAL', 'DATE', 'TIME', 'DATETIME', 'BUSDATE',
'JSON', 'STRIDED_DIM', 'FIXED_DIM',
'JSON', 'STRIDED_DIM', 'CFIXED_DIM',
'OFFSET_DIM', 'VAR_DIM', 'STRUCT',
'FIXEDSTRUCT', 'TUPLE', 'NDOBJECT',
'CSTRUCT', 'TUPLE', 'NDOBJECT',
'CONVERT', 'BYTESWAP', 'VIEW',
'CUDA_HOST', 'CUDA_DEVICE',
'PROPERTY', 'EXPR', 'UNARY_EXPR',
Expand Down Expand Up @@ -52,21 +52,23 @@
JSON = 31
STRIDED_DIM = 32
FIXED_DIM = 33
OFFSET_DIM = 34
VAR_DIM = 35
STRUCT = 36
FIXEDSTRUCT = 37
TUPLE = 38
NDOBJECT = 39
CONVERT = 40
BYTESWAP = 41
VIEW = 42
CUDA_HOST = 43
CUDA_DEVICE = 44
PROPERTY = 45
EXPR = 46
UNARY_EXPR = 47
GROUPBY = 48
TYPE = 49
CFIXED_DIM = 34
OFFSET_DIM = 35
VAR_DIM = 36
STRUCT = 37
CSTRUCT = 38
TUPLE = 39
CTUPLE = 40
NDOBJECT = 41
CONVERT = 42
BYTESWAP = 43
VIEW = 44
CUDA_HOST = 45
CUDA_DEVICE = 46
PROPERTY = 47
EXPR = 48
UNARY_EXPR = 49
GROUPBY = 50
TYPE = 51

BUILTIN_TYPE_ID_COUNT = 19
2 changes: 1 addition & 1 deletion dynd/ndt/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
make_view, \
make_unaligned, make_fixedstring, make_string, \
make_pointer, make_struct, make_cstruct, \
make_strided_dim, make_fixed_dim, make_var_dim, \
make_strided_dim, make_fixed_dim, make_cfixed_dim, make_var_dim, \
make_categorical, replace_dtype, extract_dtype, \
factor_categorical, make_bytes, make_property, \
make_reversed_property, cuda_support
Expand Down
8 changes: 4 additions & 4 deletions dynd/tests/test_array_assign.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ def test_nested_struct(self):
self.assertEqual(nd.as_py(a.z), [3j])

def test_single_struct_array(self):
a = nd.empty('3 * {x:int32, y:int32}')
a = nd.empty('3 * c{x:int32, y:int32}')
Copy link

@mrocklin mrocklin Apr 22, 2014

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We're not using this syntax in DataShape. Will this cause a mismatch?

Copy link
Member Author

@mwiebe mwiebe Apr 22, 2014

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My idea presently is to have dynd's syntax be a strict superset of datashape. In this particular case, "{x: int32, y: int32}" is a dynd type with flexible field layout in memory, while "c{x: int32, y: int32}" has a fixed field layout in memory. This is one reason why there is both "nd.type_of" and "nd.dshape_of".

a[...] = [(0,0), (3,5), (12,10)]
self.assertEqual(nd.as_py(a.x), [0, 3, 12])
self.assertEqual(nd.as_py(a.y), [0, 5, 10])
Expand All @@ -42,7 +42,7 @@ def test_single_struct_array(self):
self.assertEqual(nd.as_py(a.x), [1, 4, 14])
self.assertEqual(nd.as_py(a.y), [2, 7, 190])

a = nd.empty('2 * var * {count:int32, size:string[1,"A"]}')
a = nd.empty('2 * var * c{count:int32, size:string[1,"A"]}')
a[...] = [[(3, 'X')], [(10, 'L'), (12, 'M')]]
self.assertEqual(nd.as_py(a.count), [[3], [10, 12]])
self.assertEqual(nd.as_py(a.size), [['X'], ['L', 'M']])
Expand All @@ -61,7 +61,7 @@ def test_single_struct_array(self):
self.assertEqual(nd.as_py(a.size), [['A'], ['B', 'B']])

def test_nested_struct_array(self):
a = nd.empty('3 * {x:{a:int16, b:int16}, y:int32}')
a = nd.empty('3 * c{x:c{a:int16, b:int16}, y:int32}')
a[...] = [((0,1),0), ((2,2),5), ((100,10),10)]
self.assertEqual(nd.as_py(a.x.a), [0, 2, 100])
self.assertEqual(nd.as_py(a.x.b), [1, 2, 10])
Expand All @@ -74,7 +74,7 @@ def test_nested_struct_array(self):
self.assertEqual(nd.as_py(a.x.b), [2, 6, 110])
self.assertEqual(nd.as_py(a.y), [5, 7, 110])

a = nd.empty('2 * var * {count:int32, size:{name:string[1,"A"], id: int8}}')
a = nd.empty('2 * var * c{count:int32, size:c{name:string[1,"A"], id: int8}}')
a[...] = [[(3, ('X', 10))], [(10, ('L', 7)), (12, ('M', 5))]]
self.assertEqual(nd.as_py(a.count), [[3], [10, 12]])
self.assertEqual(nd.as_py(a.size.name), [['X'], ['L', 'M']])
Expand Down
25 changes: 15 additions & 10 deletions dynd/tests/test_array_construct.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,11 @@ def test_empty(self):
self.assertEqual(a.access_flags, 'readwrite')
self.assertEqual(nd.type_of(a), ndt.make_fixed_dim(3, ndt.int32))
self.assertEqual(a.shape, (3,))
# Constructor from type with cfixed dimension
a = nd.empty('cfixed[3] * int32')
self.assertEqual(a.access_flags, 'readwrite')
self.assertEqual(nd.type_of(a), ndt.make_cfixed_dim(3, ndt.int32))
self.assertEqual(a.shape, (3,))
# Constructor from type with fixed dimension, accesskwarg
a = nd.empty('3 * int32', access='rw')
self.assertEqual(a.access_flags, 'readwrite')
Expand Down Expand Up @@ -549,27 +554,27 @@ def test_nested_struct_array(self):

def test_missing_field(self):
self.assertRaises(RuntimeError, nd.array,
[0, 1], type='{x:int32, y:int32, z:int32}')
[0, 1], type='c{x:int32, y:int32, z:int32}')
# With dtype= parameter instead of type=
self.assertRaises(RuntimeError, nd.array,
[0, 1], dtype='{x:int32, y:int32, z:int32}')
[0, 1], dtype='c{x:int32, y:int32, z:int32}')
self.assertRaises(RuntimeError, nd.array,
{'x':0, 'z':1}, type='{x:int32, y:int32, z:int32}')
{'x':0, 'z':1}, type='c{x:int32, y:int32, z:int32}')
# With dtype= parameter instead of type=
self.assertRaises(RuntimeError, nd.array,
{'x':0, 'z':1}, dtype='{x:int32, y:int32, z:int32}')
{'x':0, 'z':1}, dtype='c{x:int32, y:int32, z:int32}')

def test_extra_field(self):
self.assertRaises(RuntimeError, nd.array,
[0, 1, 2, 3], type='{x:int32, y:int32, z:int32}')
[0, 1, 2, 3], type='c{x:int32, y:int32, z:int32}')
# With dtype= parameter instead of type=
self.assertRaises(RuntimeError, nd.array,
[0, 1, 2, 3], dtype='{x:int32, y:int32, z:int32}')
[0, 1, 2, 3], dtype='c{x:int32, y:int32, z:int32}')
self.assertRaises(RuntimeError, nd.array,
{'x':0,'y':1,'z':2,'w':3}, type='{x:int32, y:int32, z:int32}')
{'x':0,'y':1,'z':2,'w':3}, type='c{x:int32, y:int32, z:int32}')
# With dtype= parameter instead of type=
self.assertRaises(RuntimeError, nd.array,
{'x':0,'y':1,'z':2,'w':3}, dtype='{x:int32, y:int32, z:int32}')
{'x':0,'y':1,'z':2,'w':3}, dtype='c{x:int32, y:int32, z:int32}')

class TestIteratorConstruct(unittest.TestCase):
# Test dynd construction from iterators
Expand Down Expand Up @@ -731,8 +736,8 @@ def test_simple_fromiter_medsize(self):
def test_ragged_fromiter(self):
# Strided array of var from list of iterators
a = nd.array([(1+x for x in range(3)), (5*x - 10 for x in range(5)),
[2, 10]], type='M * var * int32')
self.assertEqual(nd.type_of(a), ndt.type('M * var * int32'))
[2, 10]], type='strided * var * int32')
self.assertEqual(nd.type_of(a), ndt.type('strided * var * int32'))
self.assertEqual(nd.as_py(a),
[[1,2,3], [-10, -5, 0, 5, 10], [2, 10]])
# Var array of var from iterator of iterators
Expand Down
12 changes: 6 additions & 6 deletions dynd/tests/test_array_getitem.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,10 @@ class TestArrayGetItem(unittest.TestCase):
def test_strided_dim(self):
a = nd.empty(100, ndt.int32)
a[...] = nd.range(100)
self.assertEqual(nd.type_of(a), ndt.type('A * int32'))
self.assertEqual(nd.type_of(a[...]), ndt.type('A * int32'))
self.assertEqual(nd.type_of(a), ndt.type('strided * int32'))
self.assertEqual(nd.type_of(a[...]), ndt.type('strided * int32'))
self.assertEqual(nd.type_of(a[0]), ndt.int32)
self.assertEqual(nd.type_of(a[0:1]), ndt.type('A * int32'))
self.assertEqual(nd.type_of(a[0:1]), ndt.type('strided * int32'))
self.assertEqual(nd.as_py(a[0]), 0)
self.assertEqual(nd.as_py(a[99]), 99)
self.assertEqual(nd.as_py(a[-1]), 99)
Expand All @@ -26,7 +26,7 @@ def test_fixed_dim(self):
self.assertEqual(nd.type_of(a), ndt.type('100 * int32'))
self.assertEqual(nd.type_of(a[...]), ndt.type('100 * int32'))
self.assertEqual(nd.type_of(a[0]), ndt.int32)
self.assertEqual(nd.type_of(a[0:1]), ndt.type('A * int32'))
self.assertEqual(nd.type_of(a[0:1]), ndt.type('strided * int32'))
self.assertEqual(nd.as_py(a[0]), 0)
self.assertEqual(nd.as_py(a[99]), 99)
self.assertEqual(nd.as_py(a[-1]), 99)
Expand All @@ -41,9 +41,9 @@ def test_var_dim(self):
a[...] = nd.range(100)
self.assertEqual(nd.type_of(a), ndt.type('var * int32'))
self.assertEqual(nd.type_of(a[...]), ndt.type('var * int32'))
self.assertEqual(nd.type_of(a[:]), ndt.type('M * int32'))
self.assertEqual(nd.type_of(a[:]), ndt.type('strided * int32'))
self.assertEqual(nd.type_of(a[0]), ndt.int32)
self.assertEqual(nd.type_of(a[0:1]), ndt.type('A * int32'))
self.assertEqual(nd.type_of(a[0:1]), ndt.type('strided * int32'))
self.assertEqual(nd.as_py(a[0]), 0)
self.assertEqual(nd.as_py(a[99]), 99)
self.assertEqual(nd.as_py(a[-1]), 99)
Expand Down
6 changes: 3 additions & 3 deletions dynd/tests/test_computed_fields.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ def test_simple_expr(self):
('onemore', np.int16, 'xyz + 1')])
self.assertEqual(nd.type_of(b).element_type.type_id, 'unary_expr')
self.assertEqual(nd.type_of(b).element_type.value_type,
ndt.type('{xyz: int32, twice: int32, onemore: int16}'))
ndt.type('c{xyz: int32, twice: int32, onemore: int16}'))
self.assertEqual(nd.as_py(b.xyz), [1, 2, 3, 4, 5])
self.assertEqual(nd.as_py(b.twice), [2, 4, 6, 8, 10])
self.assertEqual(nd.as_py(b.onemore), [2, 3, 4, 5, 6])
Expand All @@ -29,7 +29,7 @@ def test_rm_fields(self):
('complex', np.complex64, 'x + 1j*y')],
rm_fields=['x', 'y'])
self.assertEqual(nd.type_of(b).element_type.value_type,
ndt.type('{sum: float32, difference: float32,' +
ndt.type('c{sum: float32, difference: float32,' +
' product: float32, complex: complex[float32]}'))
self.assertEqual(nd.as_py(b.sum), [3, 0, 7]),
self.assertEqual(nd.as_py(b.difference), [-1, -2, -3])
Expand All @@ -44,7 +44,7 @@ def test_aggregate(self):
('A', 0.5, 9),
('C', 1, 5),
('B', 2, 2)],
dtype='{cat: string, x: float32, y: float32}')
dtype='c{cat: string, x: float32, y: float32}')
gb = nd.groupby(a, nd.fields(a, 'cat')).eval()
b = nd.make_computed_fields(gb, 1,
fields=[('sum_x', ndt.float32, 'sum(x)'),
Expand Down
8 changes: 4 additions & 4 deletions dynd/tests/test_ctypes_interop.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,16 +43,16 @@ class DATA(ctypes.Structure):
ndt.type(DATA))

def test_type_from_ctypes_carray(self):
self.assertEqual(ndt.make_fixed_dim(10, ndt.int32),
self.assertEqual(ndt.make_cfixed_dim(10, ndt.int32),
ndt.type(ctypes.c_int32 * 10))
self.assertEqual(ndt.make_fixed_dim((10, 3), ndt.int32),
self.assertEqual(ndt.make_cfixed_dim((10, 3), ndt.int32),
ndt.type((ctypes.c_int32 * 3) * 10))
self.assertEqual(ndt.make_fixed_dim((10, 3, 4), ndt.int32),
self.assertEqual(ndt.make_cfixed_dim((10, 3, 4), ndt.int32),
ndt.type(((ctypes.c_int32 * 4) * 3) * 10))

class POINT(ctypes.Structure):
_fields_ = [('x', ctypes.c_int32), ('y', ctypes.c_int32)]
self.assertEqual(ndt.make_fixed_dim(10, ndt.type(POINT)),
self.assertEqual(ndt.make_cfixed_dim(10, ndt.type(POINT)),
ndt.type(POINT * 10))

if __name__ == '__main__':
Expand Down
2 changes: 1 addition & 1 deletion dynd/tests/test_dtype.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -213,7 +213,7 @@ def test_type_shape(self):
# The shape attribute of ndt.type
tp = ndt.type('3 * 4 * int32')
self.assertEqual(tp.shape, (3, 4))
tp = ndt.type('M * 3 * var * int32')
tp = ndt.type('strided * 3 * var * int32')
self.assertEqual(tp.shape, (-1, 3, -1))
tp = ndt.type('var * 3 * 2 * int32')
self.assertEqual(tp.shape, (-1, 3, 2))
Expand Down
16 changes: 10 additions & 6 deletions dynd/tests/test_dtype_datashape.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,17 +34,21 @@ def test_aliases(self):
self.assertEqual(ndt.float64, ndt.type('real'))
self.assertEqual(ndt.complex_float64, ndt.type('complex'))

def test_fixed_array(self):
def test_cfixed_array(self):
# Tests of datashapes that produce the DyND fixed array type
self.assertEqual(ndt.make_fixed_dim(3, ndt.int32),
ndt.type('3 * int32'))
self.assertEqual(ndt.make_fixed_dim((5, 2), ndt.float64),
ndt.type('5 * 2 * float64'))
self.assertEqual(ndt.make_cfixed_dim(3, ndt.int32),
ndt.type('cfixed[3] * int32'))
self.assertEqual(ndt.make_cfixed_dim((5, 2), ndt.float64),
ndt.type('cfixed[5] * cfixed[2] * float64'))

def test_struct(self):
# Tests of cstruct datashape
dt = ndt.type('c{x: cfixed[3] * int32, y: string}')
self.assertEqual(dt.type_id, 'cstruct')
self.assertEqual(nd.as_py(dt.field_names), ['x', 'y'])
# Tests of struct datashape
dt = ndt.type('{x: 3 * int32, y: string}')
self.assertEqual(dt.type_id, 'cstruct')
self.assertEqual(dt.type_id, 'struct')
self.assertEqual(nd.as_py(dt.field_names), ['x', 'y'])

def test_var_dshape(self):
Expand Down
6 changes: 3 additions & 3 deletions dynd/tests/test_elwise_map.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ def multiscale(dst, src):
dst.once = src
dst.twice = [2 * nd.as_py(x) for x in src]
dst.thrice = [3 * nd.as_py(x) for x in src]
b = nd.elwise_map([a], multiscale, ndt.type('{once: int32, twice: int32, thrice: int32}'))
b = nd.elwise_map([a], multiscale, ndt.type('c{once: int32, twice: int32, thrice: int32}'))
self.assertEqual(nd.as_py(b), [{'once':0,'twice':0,'thrice':0},
{'once':1,'twice':2,'thrice':3},
{'once':2,'twice':4,'thrice':6}])
Expand Down Expand Up @@ -61,11 +61,11 @@ def computed_col(dst, src):
d.firstname = s.firstname
d.lastname = s.lastname
d.country = s.country
a = nd.parse_json('2 * {firstname: string, lastname: string, country: string}',
a = nd.parse_json('2 * c{firstname: string, lastname: string, country: string}',
"""[{"firstname":"Mike", "lastname":"Myers", "country":"Canada"},
{"firstname":"Seth", "lastname":"Green", "country":"USA"}]""")
b = nd.elwise_map([a], computed_col, ndt.type(
'{fullname: string, firstname: string, lastname: string, country: string}'))
'c{fullname: string, firstname: string, lastname: string, country: string}'))
self.assertEqual(nd.as_py(b.fullname), ['Mike Myers', 'Seth Green'])
self.assertEqual(nd.as_py(b.firstname), ['Mike', 'Seth'])
self.assertEqual(nd.as_py(b.lastname), ['Myers', 'Green'])
Expand Down
21 changes: 15 additions & 6 deletions dynd/tests/test_lowlevel.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,14 @@ def test_type_id(self):
[ndt.int32, ndt.int32], ['x', 'y'])),
_lowlevel.type_id.STRUCT)
self.assertEqual(self.type_id_of(ndt.type('{x : int32, y : int32}')),
_lowlevel.type_id.FIXEDSTRUCT)
_lowlevel.type_id.STRUCT)
self.assertEqual(self.type_id_of(ndt.type('c{x : int32, y : int32}')),
_lowlevel.type_id.CSTRUCT)
# Tuple
self.assertEqual(self.type_id_of(ndt.type('(int32, int32)')),
_lowlevel.type_id.TUPLE)
self.assertEqual(self.type_id_of(ndt.type('c(int32, int32)')),
_lowlevel.type_id.CTUPLE)
# Convert/byteswap/view
self.assertEqual(self.type_id_of(ndt.make_convert(
ndt.int32, ndt.int8)),
Expand All @@ -88,7 +95,9 @@ def test_type_id(self):
self.assertEqual(self.type_id_of(ndt.type('cuda_host[int32]')),
_lowlevel.type_id.CUDA_HOST)
# Uniform arrays
self.assertEqual(self.type_id_of(ndt.type('3 * int32')),
self.assertEqual(self.type_id_of(ndt.type('cfixed[3] * int32')),
_lowlevel.type_id.CFIXED_DIM)
self.assertEqual(self.type_id_of(ndt.type('fixed[3] * int32')),
_lowlevel.type_id.FIXED_DIM)
self.assertEqual(self.type_id_of(ndt.type('strided * int32')),
_lowlevel.type_id.STRIDED_DIM)
Expand All @@ -108,16 +117,16 @@ def test_array_from_ptr(self):
a[1] = 6
a[2] = 9
# Readwrite version
b = _lowlevel.array_from_ptr(ndt.type('3 * int32'), ctypes.addressof(a),
a, 'readwrite')
b = _lowlevel.array_from_ptr(ndt.type('cfixed[3] * int32'),
ctypes.addressof(a), a, 'readwrite')
self.assertEqual(_lowlevel.data_address_of(b), ctypes.addressof(a))
self.assertEqual(nd.dshape_of(b), '3 * int32')
self.assertEqual(nd.as_py(b), [3, 6, 9])
b[1] = 10
self.assertEqual(a[1], 10)
# Readonly version
b = _lowlevel.array_from_ptr(ndt.type('3 * int32'), ctypes.addressof(a),
a, 'readonly')
b = _lowlevel.array_from_ptr(ndt.type('cfixed[3] * int32'),
ctypes.addressof(a), a, 'readonly')
self.assertEqual(nd.as_py(b), [3, 10, 9])
def assign_to(b):
b[1] = 100
Expand Down
Loading