Skip to content

Commit 518fc8d

Browse files
JohnLCaronJohnLCaron
authored
Merge pull request #132 from JohnLCaron/fixPointerErrors
Fix FFI pointer errors
2 parents f82c28e + 5771656 commit 518fc8d

File tree

20 files changed

+219
-59
lines changed

20 files changed

+219
-59
lines changed

‎clibs/src/main/java/com/sunya/netchdf/netcdfClib/ffm/nc_vlen_t.java‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -37,10 +37,6 @@ public static MemoryLayout VlenLayout() {
3737
public static long len$get(MemorySegment seg) {
3838
return (long)constants$10.const$0.get(seg);
3939
}
40-
// added
41-
public static long getLength(MemorySegment seg, long index) {
42-
return (long)constants$10.const$0.get(seg);
43-
}
4440
/**
4541
* Setter for field:
4642
* @snippet :
@@ -53,6 +49,10 @@ public static long getLength(MemorySegment seg, long index) {
5349
public static long len$get(MemorySegment seg, long index) {
5450
return (long)constants$10.const$0.get(seg.asSlice(index*sizeof()));
5551
}
52+
// added
53+
public static long getLength(MemorySegment seg, long index) {
54+
return (long)constants$10.const$0.get(seg.asSlice(index*sizeof()));
55+
}
5656
public static void len$set(MemorySegment seg, long index, long x) {
5757
constants$10.const$0.set(seg.asSlice(index*sizeof()), x);
5858
}
@@ -68,10 +68,6 @@ public static long getLength(MemorySegment seg, long index) {
6868
public static MemorySegment p$get(MemorySegment seg) {
6969
return (java.lang.foreign.MemorySegment)constants$10.const$1.get(seg);
7070
}
71-
// added
72-
public static MemorySegment getAddress(MemorySegment seg, long index) {
73-
return (java.lang.foreign.MemorySegment)constants$10.const$1.get(seg);
74-
}
7571
/**
7672
* Setter for field:
7773
* @snippet :
@@ -84,6 +80,10 @@ public static MemorySegment getAddress(MemorySegment seg, long index) {
8480
public static MemorySegment p$get(MemorySegment seg, long index) {
8581
return (java.lang.foreign.MemorySegment)constants$10.const$1.get(seg.asSlice(index*sizeof()));
8682
}
83+
// added
84+
public static MemorySegment getAddress(MemorySegment seg, long index) {
85+
return (java.lang.foreign.MemorySegment)constants$10.const$1.get(seg.asSlice(index*sizeof()));
86+
}
8787
public static void p$set(MemorySegment seg, long index, MemorySegment x) {
8888
constants$10.const$1.set(seg.asSlice(index*sizeof()), x);
8989
}

‎clibs/src/main/kotlin/com/sunya/netchdf/hdf4Clib/H4ClibFile.kt‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -219,7 +219,10 @@ fun <T> readGRdata(grStartId: Int, grIdx: Int, datatype: Datatype<T>, wantSectio
219219
private fun <T> shapeData(datatype: Datatype<T>, values: ByteBuffer, shape: IntArray): ArrayTyped<T> {
220220
val result = when (datatype) {
221221
Datatype.BYTE -> ArrayByte(shape, values)
222-
Datatype.UBYTE, Datatype.CHAR -> ArrayUByte(shape, datatype as Datatype<UByte>, values)
222+
Datatype.UBYTE, Datatype.CHAR -> {
223+
val useShape = if (shape.size == 1 && shape[0] == 1) intArrayOf(values.limit()) else shape
224+
ArrayUByte(useShape, datatype as Datatype<UByte>, values)
225+
}
223226
Datatype.STRING -> ArrayUByte(shape, values).makeStringsFromBytes()
224227
Datatype.DOUBLE -> ArrayDouble(shape, values)
225228
Datatype.FLOAT -> ArrayFloat(shape, values)

‎clibs/src/main/kotlin/com/sunya/netchdf/hdf5Clib/H5Cbuilder.kt‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -470,15 +470,18 @@ class H5Cbuilder(val filename: String) {
470470
return results
471471
}
472472

473+
// TODO almost duplicate of H5ClibFile
474+
// internal fun readVlenStrings(session : Arena, datasetId : Long, h5ctype : H5CTypeInfo, want : Section) : ArrayString {
473475
internal fun readVlenStrings(session : Arena, attrId : Long, typeId : Long, nelems : Long) : List<String> {
474476
val strings_p: MemorySegment = session.allocateArray(ValueLayout.ADDRESS, nelems)
475477
checkErr("H5Aread VlenString", H5Aread(attrId, typeId, strings_p))
476478

477479
val slist = mutableListOf<String>()
478480
for (i in 0 until nelems) {
479-
val s2: MemorySegment = strings_p.getAtIndex(ValueLayout.ADDRESS, i)
480-
if (s2 != MemorySegment.NULL) {
481-
val value = s2.getUtf8String(0)
481+
val address: MemorySegment = strings_p.getAtIndex(ValueLayout.ADDRESS, i)
482+
if (address != MemorySegment.NULL) {
483+
val cString = address.reinterpret(Long.MAX_VALUE)
484+
val value = cString.getUtf8String(0)
482485
// val tvalue = transcodeString(value)
483486
slist.add(value)
484487
} else {
@@ -664,7 +667,8 @@ internal fun processCompoundData(session : Arena, sdataArray : ArrayStructureDat
664667
// MemorySegment get(ValueLayout.OfAddress layout, long offset) {
665668
val longAddress = bb.getLong(moffset + idx * 8)
666669
val address = MemorySegment.ofAddress(longAddress)
667-
val sval = address.getUtf8String(0)
670+
val cString = address.reinterpret(Long.MAX_VALUE)
671+
val sval = cString.getUtf8String(0)
668672
values.add(sval)
669673
}
670674
values

‎clibs/src/main/kotlin/com/sunya/netchdf/hdf5Clib/H5ClibFile.kt‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -70,9 +70,10 @@ class Hdf5ClibFile(val filename: String) : Netchdf {
7070

7171
val slist = mutableListOf<String>()
7272
for (i in 0 until nelems) {
73-
val s2: MemorySegment = strings_p.getAtIndex(ValueLayout.ADDRESS, i)
74-
if (s2 != MemorySegment.NULL) {
75-
val value = s2.getUtf8String(0)
73+
val address: MemorySegment = strings_p.getAtIndex(ValueLayout.ADDRESS, i)
74+
if (address != MemorySegment.NULL) {
75+
val cString = address.reinterpret(Long.MAX_VALUE)
76+
val value = cString.getUtf8String(0)
7677
// val tvalue = transcodeString(value)
7778
slist.add(value)
7879
} else {

‎clibs/src/main/kotlin/com/sunya/netchdf/netcdfClib/NCheader.kt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -389,7 +389,8 @@ class NCheader(val filename: String) {
389389
// val s1 = strings_p.getUtf8String(i*8) // LOOK wrong
390390
val s2: MemorySegment = strings_p.getAtIndex(ValueLayout.ADDRESS, i)
391391
if (s2 != MemorySegment.NULL) {
392-
val value = s2.getUtf8String(0)
392+
val cString = s2.reinterpret(Long.MAX_VALUE)
393+
val value = cString.getUtf8String(0)
393394
val tvalue = transcodeString(value)
394395
result.add(tvalue)
395396
} else {

‎clibs/src/main/kotlin/com/sunya/netchdf/netcdfClib/NClibFile.kt‎

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -93,11 +93,15 @@ class NClibFile(val filename: String) : Netchdf {
9393
for (elem in 0 until nelems) {
9494
val arraySize = nc_vlen_t.getLength(vlen_p, elem).toInt()
9595
val address = nc_vlen_t.getAddress(vlen_p, elem)
96-
listOfVlen.add( readVlenArray(arraySize, address, basetype))
96+
//println("elem=$elem arraySize=$arraySize address=$address")
97+
val adata = readVlenArray(arraySize, address, basetype)
98+
//println(" data=${adata.contentToString()}")
99+
listOfVlen.add( adata)
97100
}
98101
return ArrayVlen.fromArray(shape, listOfVlen, basetype) as ArrayTyped<T>
99102
// TODO nc_free_vlen(nc_vlen_t *vl);
100103
// nc_free_string(size_t len, char **data);
104+
// nc_reclaim_data()
101105
}
102106

103107
Datatype.COMPOUND -> {
@@ -114,17 +118,21 @@ class NClibFile(val filename: String) : Netchdf {
114118

115119
val members = (datatype.typedef as CompoundTypedef).members
116120
val sdataArray = ArrayStructureData(shape, bb, userType.size, members)
117-
// strings vs array of strings, also duplicate readCompoundAttValues
121+
// TODO strings vs array of strings
118122
sdataArray.putStringsOnHeap { member, offset ->
119-
val address = val_p.get(ValueLayout.ADDRESS, (offset).toLong())
120-
listOf(address.getUtf8String(0)) // LOOK not right
123+
val address = val_p.get(ADDRESS, (offset).toLong())
124+
val cString = address.reinterpret(Long.MAX_VALUE)
125+
val s = cString.getUtf8String(0)
126+
if (debugUserTypes) println("OK CompoundAttribute read string offset=$offset value=$s")
127+
listOf(s)
121128
}
122129
sdataArray.putVlensOnHeap { member, offset ->
123130
// look duplicate (maybe)
124131
val listOfVlen = mutableListOf<Array<*>>()
125132
for (elem in 0 until member.nelems) {
126-
val arraySize = val_p.get(ValueLayout.JAVA_LONG, (offset).toLong()).toInt()
127-
val address = val_p.get(ValueLayout.ADDRESS, (offset + 8).toLong())
133+
val arraySize = val_p.get(JAVA_LONG, (offset).toLong()).toInt()
134+
val zaddress = val_p.get(ValueLayout.ADDRESS, (offset + 8).toLong())
135+
val address = zaddress.reinterpret(Long.MAX_VALUE)
128136
listOfVlen.add( readVlenArray(arraySize, address, member.datatype.typedef!!.baseType))
129137
}
130138
ArrayVlen.fromArray(member.dims, listOfVlen, member.datatype.typedef!!.baseType)
@@ -283,7 +291,9 @@ class NClibFile(val filename: String) : Netchdf {
283291
nc_get_vars_string(vinfo.g4.grpid, vinfo.varid, origin_p, shape_p, stride_p, val_p))
284292
val values = mutableListOf<String>()
285293
for (i in 0 until nelems) {
286-
values.add(val_p.getAtIndex(ValueLayout.ADDRESS, i).getUtf8String(0))
294+
val address = val_p.getAtIndex(ValueLayout.ADDRESS, i)
295+
val cString = address.reinterpret(Long.MAX_VALUE)
296+
values.add(cString.getUtf8String(0))
287297
}
288298
return ArrayString(shape, values) as ArrayTyped<T>
289299
}

‎clibs/src/main/kotlin/com/sunya/netchdf/netcdfClib/UserTypes.kt‎

Lines changed: 23 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ import java.nio.ByteBuffer
1515
import java.nio.ByteOrder
1616
import java.util.*
1717

18-
private val debugUserTypes = false
18+
val debugUserTypes = false
1919

2020
@Throws(IOException::class)
2121
internal fun NCheader.readUserTypes(session: Arena, grpid: Int, gb: Group.Builder, userTypes : MutableMap<Int, UserType>) {
@@ -123,7 +123,7 @@ private fun NCheader.readCompoundFields(session: Arena, grpid: Int, typeid: Int,
123123
val ndims = ndims_p[C_INT, 0]
124124
val dims = IntArray(ndims) { dims_p.getAtIndex(ValueLayout.JAVA_INT, it.toLong())}
125125

126-
// open class StructureMember(val name: String, val datatype : Datatype, val offset: Int, val nelems : Int) {
126+
// class StructureMember(val name: String, val datatype : Datatype, val offset: Int, val nelems : Int) {
127127
val fld = StructureMember(name, convertType(ftypeid), offset.toInt(), dims)
128128
members.add(fld)
129129
if (debugUserTypes) println(" add StructureMember= $fld")
@@ -165,7 +165,11 @@ internal fun readVlenDataList(nelems : Long, basetype : Datatype<*>, vlen_p : Me
165165
val attValues = mutableListOf<List<*>>()
166166
for (elem in 0 until nelems) {
167167
val count = nc_vlen_t.getLength(vlen_p, elem)
168-
val address: MemorySegment = nc_vlen_t.getAddress(vlen_p, elem)
168+
val zaddress = nc_vlen_t.getAddress(vlen_p, elem)
169+
// val zaddress = vlen_p.get(ADDRESS, elem) // zero length memory segment
170+
// see https://stackoverflow.com/questions/77042593/indexoutofboundsexception-out-of-bound-access-on-segment-when-accessing-p
171+
val address = zaddress.reinterpret(Long.MAX_VALUE)
172+
169173
val vlenValues = mutableListOf<Any>()
170174
for (idx in 0 until count) {
171175
val value = when (basetype) {
@@ -243,20 +247,30 @@ internal fun NCheader.readCompoundAttValues(session: Arena,
243247
val val_p = session.allocate(buffSize)
244248

245249
// apparently have to call nc_get_att(), not readAttributeValues()
250+
// read data into val_p
246251
checkErr("nc_get_att", nc_get_att(grpid, varid, attname_p, val_p))
252+
// convert to byte array
247253
val raw = val_p.toArray(ValueLayout.JAVA_BYTE)
254+
// wrap in a LE ByteBuffer
248255
val bb = ByteBuffer.wrap(raw)
249256
bb.order(ByteOrder.LITTLE_ENDIAN)
250257

251258
val members = (userType.typedef as CompoundTypedef).members
252259
val sdataArray = ArrayStructureData(intArrayOf(nelems.toInt()), bb, userType.size, members)
253260
sdataArray.putStringsOnHeap { member, offset ->
254-
val address = val_p.get(ADDRESS, (offset).toLong())
255-
listOf(address.getUtf8String(0)) // LOOK not right
256-
// LOOK heres a pointer, see decodeCompoundAttData():
257-
// lval = getNativeAddr(pos, nc4bytes);
258-
// Pointer p = new Pointer(lval);
259-
// String strval = p.getString(0, CDM.UTF8);
261+
// this lamda is only called when datatype.isVlenString
262+
val address = val_p.get(ADDRESS, offset.toLong())
263+
// see https://stackoverflow.com/questions/77042593/indexoutofboundsexception-out-of-bound-access-on-segment-when-accessing-p
264+
val cString = address.reinterpret(Long.MAX_VALUE)
265+
// copy the string out of user memory onto Java heap
266+
val s = cString.getUtf8String(0)
267+
if (debugUserTypes) println("OK CompoundAttribute read string offset=$offset value=$s nelems=${member.nelems}")
268+
// TODO what about a list of strings? do we have to look at member.nelems?
269+
listOf(s)
270+
// TODO nc_free_vlen(nc_vlen_t *vl);
271+
// nc_free_string(size_t len, char **data);
272+
// nc_reclaim_data()
273+
// see https://docs.unidata.ucar.edu/netcdf-c/4.9.3/md__2home_2vagrant_2Desktop_2netcdf-c_2docs_2internal.html#intern_vlens
260274
}
261275

262276
attb.setValues(sdataArray.toList())

‎clibs/src/test/c/readVlenFloat.c‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
/*
2+
export LD_LIBRARY_PATH="/usr/lib/x86_64-linux-gnu"
3+
gcc -o testClib testClib.c -lnetcdf
4+
*/
5+
6+
#include <stdlib.h>
7+
#include <stdio.h>
8+
#include <string.h>
9+
#include <netcdf.h>
10+
11+
/* This is the name of the data file we will read. */
12+
#define FILE_NAME "/home/all/testdata/netchdf/castel/20110421-153623-snippet-VI_MB7125_01.sni"
13+
14+
/* We are reading 2D data, a 6 x 12 grid. */
15+
#define NX 6
16+
#define NY 12
17+
18+
/* Handle errors by printing an error message and exiting with a
19+
* non-zero status. */
20+
#define ERRCODE 2
21+
#define ERR(e) {printf("Error: %s\n", nc_strerror(e)); exit(ERRCODE);}
22+
23+
int
24+
main()
25+
{
26+
/* This will be the netCDF ID for the file and data variable. */
27+
int ncid, grpid, varid;
28+
29+
/* Loop indexes, and error handling. */
30+
int retval;
31+
32+
/* Open the file. NC_NOWRITE tells netCDF we want read-only access
33+
* to the file.*/
34+
if ((retval = nc_open(FILE_NAME, NC_NOWRITE, &ncid)))
35+
ERR(retval);
36+
37+
/* Get the varid of the data variable, based on its name. */
38+
if ((retval = nc_inq_varid(ncid, "mbReflectivity", &varid)))
39+
ERR(retval);
40+
printf("*** nc_inq_varid =%d\n", varid);
41+
42+
/* get the attribute unit. */
43+
int type;
44+
long attlen;
45+
if ((retval = nc_inq_att(grpid, varid, "units", &type, &attlen)))
46+
ERR(retval);
47+
printf("*** nc_inq_att type=%d attlen = %ld\n", type, attlen);
48+
/* expect type = 12, size = 1 */
49+
50+
char **string_attr = (char**)malloc(attlen * sizeof(char*));
51+
memset(string_attr, 0, attlen * sizeof(char*));
52+
53+
if ((retval = nc_get_att_string(grpid, varid, "units", string_attr)))
54+
ERR(retval);
55+
for (size_t k = 0; k < attlen; ++k) {
56+
printf(" CAPE:units[%ld] = '%s'\n", k, string_attr[k]);
57+
}
58+
59+
if ((retval = nc_free_string(attlen, string_attr)))
60+
ERR(retval);
61+
free(string_attr);
62+
63+
/* Close the file, freeing all resources. */
64+
if ((retval = nc_close(ncid)))
65+
ERR(retval);
66+
67+
printf("*** SUCCESS reading example file %s!\n", FILE_NAME);
68+
return 0;
69+
}

‎clibs/src/test/c/testClib.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ gcc -o testClib testClib.c -lnetcdf
99
#include <netcdf.h>
1010

1111
/* This is the name of the data file we will read. */
12-
#define FILE_NAME "/home/oem/testdata/netchdf/joleenf/IASI_20120229022657Z.atm_prof_rtv.h5"
12+
#define FILE_NAME "/home/all/testdata/netchdf/joleenf/IASI_20120229022657Z.atm_prof_rtv.h5"
1313

1414
/* We are reading 2D data, a 6 x 12 grid. */
1515
#define NX 6

‎clibs/src/test/kotlin/com/sunya/netchdf/NetchdfClibExtra.kt‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
package com.sunya.netchdf
22

3-
import com.sunya.cdm.api.*
43
import com.sunya.cdm.util.Stats
54
import com.sunya.testdata.NetchdfExtraFiles
65
import org.junit.jupiter.api.Disabled
@@ -10,7 +9,6 @@ import org.junit.jupiter.params.provider.Arguments
109
import org.junit.jupiter.params.provider.MethodSource
1110
import com.sunya.testdata.testData
1211
import org.junit.jupiter.api.AfterAll
13-
import java.util.*
1412
import java.util.stream.Stream
1513

1614
// Compare header using cdl(!strict) with Netchdf and NetcdfClibFile
@@ -50,7 +48,7 @@ class NetchdfClibExtra {
5048

5149
// this one we could probably fix
5250
@Test
53-
@Disabled
51+
// @Disabled
5452
fun unsolved2() {
5553
val filename = testData + "netchdf/tomas/S3A_OL_CCDB_CHAR_AllFiles.20101019121929_1.nc4"
5654
// showMyHeader(filename)
@@ -60,6 +58,7 @@ class NetchdfClibExtra {
6058
//readDataCompareNC(filename)
6159
}
6260

61+
6362
///////////////////////////////////////////////////////
6463
@ParameterizedTest
6564
@MethodSource("params")

0 commit comments

Comments
 (0)