001    /*
002     * Copyright (C) 2008-2010 by Holger Arndt
003     *
004     * This file is part of the Universal Java Matrix Package (UJMP).
005     * See the NOTICE file distributed with this work for additional
006     * information regarding copyright ownership and licensing.
007     *
008     * UJMP is free software; you can redistribute it and/or modify
009     * it under the terms of the GNU Lesser General Public License as
010     * published by the Free Software Foundation; either version 2
011     * of the License, or (at your option) any later version.
012     *
013     * UJMP is distributed in the hope that it will be useful,
014     * but WITHOUT ANY WARRANTY; without even the implied warranty of
015     * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
016     * GNU Lesser General Public License for more details.
017     *
018     * You should have received a copy of the GNU Lesser General Public
019     * License along with UJMP; if not, write to the
020     * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
021     * Boston, MA  02110-1301  USA
022     */
023    
024    package org.ujmp.core.doublematrix.impl;
025    
026    import java.io.Closeable;
027    import java.io.File;
028    import java.io.FileNotFoundException;
029    import java.io.IOException;
030    import java.io.ObjectInputStream;
031    import java.io.ObjectOutputStream;
032    import java.io.OptionalDataException;
033    import java.nio.ByteBuffer;
034    import java.nio.ByteOrder;
035    
036    import org.ujmp.core.Coordinates;
037    import org.ujmp.core.Matrix;
038    import org.ujmp.core.doublematrix.stub.AbstractDenseDoubleMatrix;
039    import org.ujmp.core.exceptions.MatrixException;
040    import org.ujmp.core.interfaces.Erasable;
041    import org.ujmp.core.util.MathUtil;
042    import org.ujmp.core.util.io.BufferedRandomAccessFile;
043    
044    public class DenseFileMatrix extends AbstractDenseDoubleMatrix implements Erasable, Closeable {
045            private static final long serialVersionUID = 1754729146021609978L;
046    
047            private transient BufferedRandomAccessFile randomAccessFile = null;
048    
049            private int bufferSize = 65536;
050    
051            public static final int BYTE = 0;
052    
053            public static final int CHAR = 1;
054    
055            public static final int DOUBLE = 2;
056    
057            public static final int FLOAT = 3;
058    
059            public static final int INT = 4;
060    
061            public static final int LONG = 5;
062    
063            public static final int SHORT = 6;
064    
065            public static final int UNSIGNEDBYTE = 7;
066    
067            public static final int UNSIGNEDSHORT = 8;
068    
069            public static final int SHORTLITTLEENDIAN = 9;
070    
071            public static final int INTLITTLEENDIAN = 10;
072    
073            public static final int LONGLITTLEENDIAN = 11;
074    
075            public static final int BOOLEAN = 12;
076    
077            private int dataType = DOUBLE;
078    
079            private File file = null;
080    
081            private long[] size;
082    
083            private long offset = 0;
084    
085            private int bitsPerValue = 1;
086    
087            private boolean readOnly = false;
088    
089            private static ByteOrder byteOrder = ByteOrder.BIG_ENDIAN;
090    
091            public DenseFileMatrix(File file, long... size) throws IOException {
092                    this(file, 0, DOUBLE, false, size);
093            }
094    
095            public DenseFileMatrix(File file) throws IOException {
096                    this(file, 0, BYTE, true, file.length(), 1);
097            }
098    
099            public DenseFileMatrix(File file, int dataType, long... size) throws IOException {
100                    this(file, 0, dataType, false, size);
101            }
102    
103            public DenseFileMatrix(File file, long offset, int dataType, boolean readOnly, long... size)
104                            throws IOException {
105                    this(65536, file, offset, dataType, readOnly, size);
106            }
107    
108            public DenseFileMatrix(int bufferSize, File file, long offset, int dataType,
109                            boolean readOnly, long... size) throws IOException {
110                    if (file == null) {
111                            file = File.createTempFile("denseFileMatrix", ".dat");
112                            file.deleteOnExit();
113                    }
114                    this.bufferSize = bufferSize;
115                    this.file = file;
116                    this.size = size;
117                    this.offset = offset;
118                    this.dataType = dataType;
119                    this.bitsPerValue = getBitsPerValue(dataType);
120                    this.readOnly = readOnly;
121            }
122    
123            public int getBufferSize() {
124                    return bufferSize;
125            }
126    
127            public void setBufferSize(int bufferSize) {
128                    this.bufferSize = bufferSize;
129            }
130    
131            private void createFile() {
132                    try {
133                            if (readOnly) {
134                                    randomAccessFile = new BufferedRandomAccessFile(file, "r", bufferSize);
135                            } else {
136                                    try {
137                                            randomAccessFile = new BufferedRandomAccessFile(file, "rw", bufferSize);
138                                    } catch (FileNotFoundException e) {
139                                            randomAccessFile = new BufferedRandomAccessFile(file, "r", bufferSize);
140                                    }
141                                    long difference = getFileLength() + offset - randomAccessFile.length();
142                                    if (difference > 0) {
143                                            long seek = (long) (Math.ceil(getPos(getRowCount(), getColumnCount())));
144                                            switch (dataType) {
145                                            case BYTE:
146                                                    randomAccessFile.writeByte(seek, (byte) 0.0);
147                                                    break;
148                                            case CHAR:
149                                                    randomAccessFile.writeChar(seek, (char) 0.0);
150                                                    break;
151                                            case DOUBLE:
152                                                    randomAccessFile.writeDouble(seek, 0.0);
153                                                    break;
154                                            case FLOAT:
155                                                    randomAccessFile.writeFloat(seek, (float) 0.0);
156                                                    break;
157                                            case SHORT:
158                                                    randomAccessFile.writeShort(seek, (short) 0.0);
159                                                    break;
160                                            case INT:
161                                                    randomAccessFile.writeInt(seek, (int) 0.0);
162                                                    break;
163                                            case LONG:
164                                                    randomAccessFile.writeLong(seek, (long) 0.0);
165                                                    break;
166                                            case UNSIGNEDBYTE:
167                                                    randomAccessFile.writeByte(seek, (byte) 0.0);
168                                                    break;
169                                            case UNSIGNEDSHORT:
170                                                    randomAccessFile.writeShort(seek, (short) 0.0);
171                                                    break;
172                                            case INTLITTLEENDIAN:
173                                                    randomAccessFile.writeInt(seek, (int) 0.0);
174                                                    break;
175                                            case SHORTLITTLEENDIAN:
176                                                    randomAccessFile.writeInt(seek, (short) 0.0);
177                                                    break;
178                                            case LONGLITTLEENDIAN:
179                                                    randomAccessFile.writeLong(seek, (long) 0.0);
180                                                    break;
181                                            case BOOLEAN:
182                                                    randomAccessFile.writeByte(seek, (byte) 0.0);
183                                                    break;
184                                            }
185                                    }
186                            }
187                    } catch (Exception e) {
188                            throw new MatrixException("could not open file", e);
189                    }
190            }
191    
192            public DenseFileMatrix(long... size) throws IOException {
193                    this(null, size);
194            }
195    
196            public DenseFileMatrix(Matrix m) throws IOException {
197                    this(m.getSize());
198                    for (long[] c : m.allCoordinates()) {
199                            setAsDouble(m.getAsDouble(c), c);
200                    }
201            }
202    
203            public BufferedRandomAccessFile getRandomAccessFile() {
204                    return randomAccessFile;
205            }
206    
207            public File getFile() {
208                    return file;
209            }
210    
211            private static final int getBitsPerValue(int dataType) {
212                    switch (dataType) {
213                    case BYTE:
214                            return 8;
215                    case CHAR:
216                            return 8;
217                    case DOUBLE:
218                            return 64;
219                    case FLOAT:
220                            return 32;
221                    case INT:
222                            return 32;
223                    case INTLITTLEENDIAN:
224                            return 32;
225                    case LONG:
226                            return 64;
227                    case LONGLITTLEENDIAN:
228                            return 64;
229                    case SHORT:
230                            return 16;
231                    case UNSIGNEDBYTE:
232                            return 8;
233                    case UNSIGNEDSHORT:
234                            return 16;
235                    case SHORTLITTLEENDIAN:
236                            return 16;
237                    case BOOLEAN:
238                            return 1;
239                    default:
240                            return 32;
241                    }
242            }
243    
244            public long getBytesPerValue() {
245                    return getBitsPerValue() / 8;
246            }
247    
248            public int getBitsPerValue() {
249                    return bitsPerValue;
250            }
251    
252            private long getPos(long... pos) {
253                    if (getBytesPerValue() < 1) {
254                            throw new RuntimeException("not supported");
255                    }
256                    return MathUtil.pos2IndexRowMajor(size, pos) * getBytesPerValue() + offset;
257            }
258    
259            public long getFileLength() {
260                    double prod = (double) getRowCount() * (double) getColumnCount() * getBytesPerValue();
261                    return (long) Math.ceil(prod);
262            }
263    
264            public int getDataType() {
265                    return dataType;
266            }
267    
268            public synchronized double getDouble(long... c) {
269                    if (randomAccessFile == null) {
270                            createFile();
271                    }
272                    if (randomAccessFile != null) {
273                            try {
274                                    long seek = getPos(c);
275    
276                                    if (seek < getFileLength() + offset) {
277    
278                                            byte[] bytes = null;
279    
280                                            switch (getDataType()) {
281                                            case BYTE:
282                                                    bytes = new byte[1];
283                                                    randomAccessFile.read(seek, bytes);
284                                                    return bytes[0];
285                                            case CHAR:
286                                                    bytes = new byte[1];
287                                                    randomAccessFile.read(seek, bytes);
288                                                    return bytes[0];
289                                            case DOUBLE:
290                                                    bytes = new byte[8];
291                                                    randomAccessFile.read(seek, bytes);
292                                                    return ByteBuffer.wrap(bytes).order(byteOrder).getDouble();
293                                            case FLOAT:
294                                                    bytes = new byte[4];
295                                                    randomAccessFile.read(seek, bytes);
296                                                    return ByteBuffer.wrap(bytes).order(byteOrder).getFloat();
297                                            case SHORT:
298                                                    bytes = new byte[2];
299                                                    randomAccessFile.read(seek, bytes);
300                                                    return ByteBuffer.wrap(bytes).order(byteOrder).getShort();
301                                            case INT:
302                                                    bytes = new byte[4];
303                                                    randomAccessFile.read(seek, bytes);
304                                                    return ByteBuffer.wrap(bytes).order(byteOrder).getInt();
305                                            case LONG:
306                                                    bytes = new byte[8];
307                                                    randomAccessFile.read(seek, bytes);
308                                                    return ByteBuffer.wrap(bytes).order(byteOrder).getLong();
309                                            case LONGLITTLEENDIAN:
310                                                    bytes = new byte[8];
311                                                    randomAccessFile.read(seek, bytes);
312                                                    return ByteBuffer.wrap(bytes).order(byteOrder).getLong();
313                                            case SHORTLITTLEENDIAN:
314                                                    bytes = new byte[2];
315                                                    randomAccessFile.read(seek, bytes);
316                                                    return ByteBuffer.wrap(bytes).order(byteOrder).getShort();
317                                            case INTLITTLEENDIAN:
318                                                    bytes = new byte[4];
319                                                    randomAccessFile.read(seek, bytes);
320                                                    return ByteBuffer.wrap(bytes).order(byteOrder).getInt();
321                                            case UNSIGNEDBYTE:
322                                                    return randomAccessFile.readUnsignedByte();
323                                            case UNSIGNEDSHORT:
324                                                    return randomAccessFile.readUnsignedShort();
325                                            case BOOLEAN:
326                                                    // return getBit(randomAccessFile.readByte(), pos -
327                                                    // Math.floor(pos));
328                                            }
329    
330                                    } else {
331                                            throw new MatrixException("no such coordinates: " + Coordinates.toString(c));
332                                    }
333                            } catch (Exception e) {
334                                    throw new MatrixException("could not read value", e);
335                            }
336                    }
337                    return 0.0;
338            }
339    
340            public void setSize(long... size) {
341                    this.size = size;
342            }
343    
344            private static final double getBit(byte b, double offset) {
345                    if (offset == 0.0) {
346                            return ((b & 0x01) > 0) ? 1 : 0;
347                    } else if (offset == 0.125) {
348                            return ((b & 0x02) > 0) ? 1 : 0;
349                    } else if (offset == 0.25) {
350                            return ((b & 0x04) > 0) ? 1 : 0;
351                    } else if (offset == 0.375) {
352                            return ((b & 0x08) > 0) ? 1 : 0;
353                    } else if (offset == 0.5) {
354                            return ((b & 0x10) > 0) ? 1 : 0;
355                    } else if (offset == 0.625) {
356                            return ((b & 0x20) > 0) ? 1 : 0;
357                    } else if (offset == 0.75) {
358                            return ((b & 0x40) > 0) ? 1 : 0;
359                    } else if (offset == 0.875) {
360                            return ((b & 0x80) > 0) ? 1 : 0;
361                    }
362                    return 0.0;
363            }
364    
365            private static final byte setBit(byte b, double offset) {
366                    if (offset == 0.0) {
367                            b = (byte) (b | 0x01);
368                    } else if (offset == 0.125) {
369                            b = (byte) (b | 0x02);
370                    } else if (offset == 0.25) {
371                            b = (byte) (b | 0x04);
372                    } else if (offset == 0.375) {
373                            b = (byte) (b | 0x08);
374                    } else if (offset == 0.5) {
375                            b = (byte) (b | 0x10);
376                    } else if (offset == 0.625) {
377                            b = (byte) (b | 0x20);
378                    } else if (offset == 0.75) {
379                            b = (byte) (b | 0x40);
380                    } else if (offset == 0.875) {
381                            b = (byte) (b | 0x80);
382                    }
383                    return b;
384            }
385    
386            public synchronized void setDouble(double value, long... c) {
387                    if (isReadOnly())
388                            return;
389    
390                    try {
391    
392                            if (file == null) {
393                                    file = File.createTempFile("matrix", null);
394                            }
395    
396                            if (randomAccessFile == null) {
397                                    createFile();
398                            }
399    
400                            long seek = getPos(c);
401    
402                            ByteBuffer bb = null;
403    
404                            switch (dataType) {
405                            case BYTE:
406                                    randomAccessFile.writeByte(seek, (byte) value);
407                                    break;
408                            case CHAR:
409                                    randomAccessFile.writeChar(seek, (char) value);
410                                    break;
411                            case DOUBLE:
412                                    randomAccessFile.writeDouble(seek, value);
413                                    break;
414                            case FLOAT:
415                                    randomAccessFile.writeFloat(seek, (float) value);
416                                    break;
417                            case SHORT:
418                                    randomAccessFile.writeShort(seek, (short) value);
419                                    break;
420                            case SHORTLITTLEENDIAN:
421                                    bb = ByteBuffer.allocate(2).order(byteOrder);
422                                    bb.putShort((short) value);
423                                    randomAccessFile.write(seek, bb.array());
424                                    break;
425                            case INT:
426                                    randomAccessFile.writeInt(seek, (int) value);
427                                    break;
428                            case INTLITTLEENDIAN:
429                                    bb = ByteBuffer.allocate(4).order(byteOrder);
430                                    bb.putInt((int) value);
431                                    randomAccessFile.write(seek, bb.array());
432                                    break;
433                            case LONG:
434                                    randomAccessFile.writeLong(seek, (long) value);
435                                    break;
436                            case LONGLITTLEENDIAN:
437                                    bb = ByteBuffer.allocate(8).order(byteOrder);
438                                    bb.putLong((long) value);
439                                    randomAccessFile.write(seek, bb.array());
440                                    break;
441                            case UNSIGNEDBYTE:
442                                    randomAccessFile.writeByte(seek, (byte) value);
443                                    break;
444                            case UNSIGNEDSHORT:
445                                    randomAccessFile.writeShort(seek, (short) value);
446                                    break;
447                            case BOOLEAN:
448                                    throw new IOException("not supported");
449                            }
450                    } catch (Exception e) {
451                            throw new MatrixException("could not write value at coordinates "
452                                            + Coordinates.toString(c), e);
453                    }
454            }
455    
456            protected void finalize() throws Throwable {
457                    super.finalize();
458                    if (randomAccessFile != null) {
459                            try {
460                                    randomAccessFile.close();
461                                    randomAccessFile = null;
462                            } catch (Throwable e) {
463                            }
464                    }
465            }
466    
467            public long[] getSize() {
468                    return size;
469            }
470    
471            public boolean isReadOnly() {
472                    return readOnly;
473            }
474    
475            public static final int getShortLittleEndian(byte[] bytes) {
476                    return ByteBuffer.wrap(bytes).order(byteOrder).getShort();
477            }
478    
479            public static final int getIntLittleEndian(byte[] bytes) {
480                    return ByteBuffer.wrap(bytes).order(byteOrder).getInt();
481            }
482    
483            private void writeObject(ObjectOutputStream s) throws IOException {
484                    s.defaultWriteObject();
485                    for (long[] c : availableCoordinates()) {
486                            s.writeObject(new Coordinates(c));
487                            s.writeObject(getDouble(c));
488                    }
489            }
490    
491            private void readObject(ObjectInputStream s) throws IOException, ClassNotFoundException {
492                    s.defaultReadObject();
493                    while (true) {
494                            try {
495                                    Coordinates c = (Coordinates) s.readObject();
496                                    Double o = (Double) s.readObject();
497                                    setDouble(o, c.co);
498                            } catch (OptionalDataException e) {
499                                    return;
500                            }
501                    }
502            }
503    
504            public void erase() throws IOException {
505                    close();
506                    file.delete();
507            }
508    
509            public void close() throws IOException {
510                    if (randomAccessFile != null) {
511                            randomAccessFile.close();
512                    }
513    
514            }
515    
516    }