001 /* 002 * Copyright (C) 2008-2010 by Holger Arndt 003 * 004 * This file is part of the Universal Java Matrix Package (UJMP). 005 * See the NOTICE file distributed with this work for additional 006 * information regarding copyright ownership and licensing. 007 * 008 * UJMP is free software; you can redistribute it and/or modify 009 * it under the terms of the GNU Lesser General Public License as 010 * published by the Free Software Foundation; either version 2 011 * of the License, or (at your option) any later version. 012 * 013 * UJMP is distributed in the hope that it will be useful, 014 * but WITHOUT ANY WARRANTY; without even the implied warranty of 015 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 016 * GNU Lesser General Public License for more details. 017 * 018 * You should have received a copy of the GNU Lesser General Public 019 * License along with UJMP; if not, write to the 020 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, 021 * Boston, MA 02110-1301 USA 022 */ 023 024 package org.ujmp.hadoop; 025 026 import java.io.Closeable; 027 import java.io.IOException; 028 import java.io.Serializable; 029 import java.util.Collection; 030 import java.util.Map; 031 import java.util.Set; 032 033 import org.apache.hadoop.conf.Configuration; 034 import org.apache.hadoop.fs.FileSystem; 035 import org.apache.hadoop.fs.Path; 036 import org.apache.hadoop.io.MapFile; 037 import org.apache.hadoop.io.Text; 038 import org.ujmp.core.util.ReflectionUtil; 039 import org.ujmp.core.util.SerializationUtil; 040 041 public class HadoopMap<K, V> implements Map<K, V>, Closeable { 042 043 private MapFile.Reader reader = null; 044 private MapFile.Writer writer = null; 045 046 private Configuration conf = null; 047 private Path dirName = null; 048 private FileSystem fs = null; 049 private Path qualifiedDirName = null; 050 051 public HadoopMap() throws IOException { 052 conf = new Configuration(); 053 dirName = new Path("/tmp/test.hadoop"); 054 fs = FileSystem.getLocal(conf); 055 qualifiedDirName = fs.makeQualified(dirName); 056 MapFile.Writer.setIndexInterval(conf, 3); 057 } 058 059 060 public void clear() { 061 // TODO Auto-generated method stub 062 } 063 064 private void prepareReader() throws IOException { 065 if (writer != null) { 066 writer.close(); 067 writer = null; 068 } 069 if (reader == null) { 070 reader = new MapFile.Reader(fs, qualifiedDirName.toString(), conf); 071 } 072 } 073 074 private void prepareWriter() throws IOException { 075 if (reader != null) { 076 reader.close(); 077 reader = null; 078 } 079 if (writer == null) { 080 writer = new MapFile.Writer(conf, fs, qualifiedDirName.toString(), 081 Text.class, Text.class); 082 } 083 } 084 085 086 public boolean containsKey(Object key) { 087 // TODO Auto-generated method stub 088 return false; 089 } 090 091 092 public boolean containsValue(Object value) { 093 // TODO Auto-generated method stub 094 return false; 095 } 096 097 098 public Set<java.util.Map.Entry<K, V>> entrySet() { 099 // TODO Auto-generated method stub 100 return null; 101 } 102 103 104 public V get(Object key) { 105 try { 106 prepareReader(); 107 Text k = new Text(SerializationUtil.serialize((Serializable) key)); 108 Text v = new Text(); 109 Text t = (Text) reader.get(k, v); 110 if (t == null || t.getBytes() == null || t.getBytes().length == 0) { 111 return null; 112 } 113 return (V) SerializationUtil.deserialize(t.getBytes()); 114 } catch (Exception e) { 115 throw new RuntimeException("could not get value for key: " + key, e); 116 } 117 } 118 119 120 public boolean isEmpty() { 121 return size() == 0; 122 } 123 124 125 public Set<K> keySet() { 126 // TODO Auto-generated method stub 127 return null; 128 } 129 130 131 public V put(K key, V value) { 132 try { 133 prepareWriter(); 134 Text k = new Text(SerializationUtil.serialize((Serializable) key)); 135 Text v = new Text(SerializationUtil.serialize((Serializable) value)); 136 writer.append(k, v); 137 return null; 138 } catch (Exception e) { 139 throw new RuntimeException("could not store value: " + key + ", " 140 + value, e); 141 } 142 } 143 144 145 public void putAll(Map<? extends K, ? extends V> m) { 146 for (K k : m.keySet()) { 147 put(k, m.get(k)); 148 } 149 150 } 151 152 153 public V remove(Object key) { 154 // TODO Auto-generated method stub 155 return null; 156 } 157 158 159 public int size() { 160 try { 161 prepareWriter(); 162 return (int) (long) (Long) ReflectionUtil.extractPrivateField( 163 MapFile.Writer.class, writer, "size"); 164 } catch (Exception e) { 165 throw new RuntimeException("could not query size", e); 166 } 167 } 168 169 170 public Collection<V> values() { 171 return null; 172 } 173 174 175 public void close() throws IOException { 176 if (reader != null) { 177 reader.close(); 178 } 179 if (writer != null) { 180 writer.close(); 181 } 182 } 183 184 public static void main(String[] args) throws Exception { 185 Map<Object, Object> map = new HadoopMap<Object, Object>(); 186 System.out.println(map.size()); 187 map.put("test", "test"); 188 System.out.println(map.size()); 189 map.put("a", "a"); 190 System.out.println(map.get("test")); 191 System.out.println(map.get("test2")); 192 } 193 194 }