001    /*
002     * Copyright (C) 2008-2010 by Holger Arndt
003     *
004     * This file is part of the Universal Java Matrix Package (UJMP).
005     * See the NOTICE file distributed with this work for additional
006     * information regarding copyright ownership and licensing.
007     *
008     * UJMP is free software; you can redistribute it and/or modify
009     * it under the terms of the GNU Lesser General Public License as
010     * published by the Free Software Foundation; either version 2
011     * of the License, or (at your option) any later version.
012     *
013     * UJMP is distributed in the hope that it will be useful,
014     * but WITHOUT ANY WARRANTY; without even the implied warranty of
015     * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
016     * GNU Lesser General Public License for more details.
017     *
018     * You should have received a copy of the GNU Lesser General Public
019     * License along with UJMP; if not, write to the
020     * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
021     * Boston, MA  02110-1301  USA
022     */
023    
024    package org.ujmp.hadoop;
025    
026    import java.io.Closeable;
027    import java.io.IOException;
028    import java.io.Serializable;
029    import java.util.Collection;
030    import java.util.Map;
031    import java.util.Set;
032    
033    import org.apache.hadoop.conf.Configuration;
034    import org.apache.hadoop.fs.FileSystem;
035    import org.apache.hadoop.fs.Path;
036    import org.apache.hadoop.io.MapFile;
037    import org.apache.hadoop.io.Text;
038    import org.ujmp.core.util.ReflectionUtil;
039    import org.ujmp.core.util.SerializationUtil;
040    
041    public class HadoopMap<K, V> implements Map<K, V>, Closeable {
042    
043            private MapFile.Reader reader = null;
044            private MapFile.Writer writer = null;
045    
046            private Configuration conf = null;
047            private Path dirName = null;
048            private FileSystem fs = null;
049            private Path qualifiedDirName = null;
050    
051            public HadoopMap() throws IOException {
052                    conf = new Configuration();
053                    dirName = new Path("/tmp/test.hadoop");
054                    fs = FileSystem.getLocal(conf);
055                    qualifiedDirName = fs.makeQualified(dirName);
056                    MapFile.Writer.setIndexInterval(conf, 3);
057            }
058    
059            
060            public void clear() {
061                    // TODO Auto-generated method stub
062            }
063    
064            private void prepareReader() throws IOException {
065                    if (writer != null) {
066                            writer.close();
067                            writer = null;
068                    }
069                    if (reader == null) {
070                            reader = new MapFile.Reader(fs, qualifiedDirName.toString(), conf);
071                    }
072            }
073    
074            private void prepareWriter() throws IOException {
075                    if (reader != null) {
076                            reader.close();
077                            reader = null;
078                    }
079                    if (writer == null) {
080                            writer = new MapFile.Writer(conf, fs, qualifiedDirName.toString(),
081                                            Text.class, Text.class);
082                    }
083            }
084    
085            
086            public boolean containsKey(Object key) {
087                    // TODO Auto-generated method stub
088                    return false;
089            }
090    
091            
092            public boolean containsValue(Object value) {
093                    // TODO Auto-generated method stub
094                    return false;
095            }
096    
097            
098            public Set<java.util.Map.Entry<K, V>> entrySet() {
099                    // TODO Auto-generated method stub
100                    return null;
101            }
102    
103            
104            public V get(Object key) {
105                    try {
106                            prepareReader();
107                            Text k = new Text(SerializationUtil.serialize((Serializable) key));
108                            Text v = new Text();
109                            Text t = (Text) reader.get(k, v);
110                            if (t == null || t.getBytes() == null || t.getBytes().length == 0) {
111                                    return null;
112                            }
113                            return (V) SerializationUtil.deserialize(t.getBytes());
114                    } catch (Exception e) {
115                            throw new RuntimeException("could not get value for key: " + key, e);
116                    }
117            }
118    
119            
120            public boolean isEmpty() {
121                    return size() == 0;
122            }
123    
124            
125            public Set<K> keySet() {
126                    // TODO Auto-generated method stub
127                    return null;
128            }
129    
130            
131            public V put(K key, V value) {
132                    try {
133                            prepareWriter();
134                            Text k = new Text(SerializationUtil.serialize((Serializable) key));
135                            Text v = new Text(SerializationUtil.serialize((Serializable) value));
136                            writer.append(k, v);
137                            return null;
138                    } catch (Exception e) {
139                            throw new RuntimeException("could not store value: " + key + ", "
140                                            + value, e);
141                    }
142            }
143    
144            
145            public void putAll(Map<? extends K, ? extends V> m) {
146                    for (K k : m.keySet()) {
147                            put(k, m.get(k));
148                    }
149    
150            }
151    
152            
153            public V remove(Object key) {
154                    // TODO Auto-generated method stub
155                    return null;
156            }
157    
158            
159            public int size() {
160                    try {
161                            prepareWriter();
162                            return (int) (long) (Long) ReflectionUtil.extractPrivateField(
163                                            MapFile.Writer.class, writer, "size");
164                    } catch (Exception e) {
165                            throw new RuntimeException("could not query size", e);
166                    }
167            }
168    
169            
170            public Collection<V> values() {
171                    return null;
172            }
173    
174            
175            public void close() throws IOException {
176                    if (reader != null) {
177                            reader.close();
178                    }
179                    if (writer != null) {
180                            writer.close();
181                    }
182            }
183    
184            public static void main(String[] args) throws Exception {
185                    Map<Object, Object> map = new HadoopMap<Object, Object>();
186                    System.out.println(map.size());
187                    map.put("test", "test");
188                    System.out.println(map.size());
189                    map.put("a", "a");
190                    System.out.println(map.get("test"));
191                    System.out.println(map.get("test2"));
192            }
193    
194    }