diff --git a/Homework1/pom.xml b/Homework1/pom.xml index 6362804..cb5f3d2 100644 --- a/Homework1/pom.xml +++ b/Homework1/pom.xml @@ -125,7 +125,7 @@ host="${remote.host.ip}" username="${remote.host.name}" password="${remote.host.password}" - command="chmod -R 777 /root" + command="chmod -R 777 /root/${job.executor.script.name}" timeout="120000" usepty="true" /> diff --git a/Homework2/pom.xml b/Homework2/pom.xml index 51f0f82..614dc2f 100644 --- a/Homework2/pom.xml +++ b/Homework2/pom.xml @@ -167,7 +167,7 @@ host="${remote.host.ip}" username="${remote.host.name}" password="${remote.host.password}" - command="chmod -R 777 /root" + command="chmod -R 777 /root/${job.executor.script.name}" timeout="120000" usepty="true" /> diff --git a/Homework2/src/main/java/ru/hokan/FileProcessor.java b/Homework2/src/main/java/ru/hokan/FileProcessor.java index 83631b9..01f0af0 100644 --- a/Homework2/src/main/java/ru/hokan/FileProcessor.java +++ b/Homework2/src/main/java/ru/hokan/FileProcessor.java @@ -23,7 +23,7 @@ public class FileProcessor implements Callable { private final FileSystem fileSystem; private final Configuration configuration; private int processingRecordsBlock = 0; - private Map resultMap = new HashMap(NUMBER_OF_PROCESSING_RECORDS); + private Map resultMap = new TreeMap(); private final String outputURI; public FileProcessor(LocatedFileStatus fileStatus, FileSystem fileSystem, Configuration configuration, String outputURI) { @@ -96,9 +96,8 @@ private void processFile(InputStream stream) { } private void dropDataToHDFS() { - List> entries = SortUtils.sortMapByValuesDescending(resultMap); try { - writeRecordsToHDFS(entries); + writeRecordsToHDFS(resultMap); } catch (URISyntaxException e) { LOGGER.error(e.getMessage(), e); } catch (IOException e) { @@ -106,7 +105,7 @@ private void dropDataToHDFS() { } } - private void writeRecordsToHDFS(List> entries) throws URISyntaxException, IOException { + private void writeRecordsToHDFS(Map entries) throws URISyntaxException, IOException { String threadName = Thread.currentThread().getName(); String blockFileName = threadName + "-block-" + processingRecordsBlock; Configuration configuration = new Configuration(); @@ -119,7 +118,7 @@ private void writeRecordsToHDFS(List> entries) throws OutputStream outputStream = fileSystem.create(filePath); BufferedWriter bufferedWriter = new BufferedWriter(new OutputStreamWriter(outputStream, "UTF-8")); LOGGER.info("Writing into \'" + blockFileName + "\' from \'" + threadName + "\' thread ..."); - for (Map.Entry stringIntegerEntry : entries) { + for (Map.Entry stringIntegerEntry : entries.entrySet()) { bufferedWriter.write(stringIntegerEntry.getKey() + "\t" + stringIntegerEntry.getValue()); bufferedWriter.write('\n'); } diff --git a/Homework2/src/main/java/ru/hokan/PathFilesProcessor.java b/Homework2/src/main/java/ru/hokan/PathFilesProcessor.java index ddba9d9..b47e3c7 100644 --- a/Homework2/src/main/java/ru/hokan/PathFilesProcessor.java +++ b/Homework2/src/main/java/ru/hokan/PathFilesProcessor.java @@ -44,23 +44,48 @@ private void processFiles(Configuration configuration, FileSystem fileSystem, Re LOGGER.info("Size of readers: " + readers.size()); - TreeMap currentTopRecordsFromBuffers = createBufferedReaderToLineMap(readers); + List lineList = new ArrayList(); + List bufferedReaders = new ArrayList(); + for (BufferedReader reader : readers) { + String s = reader.readLine(); + String[] split = s.split("\\t"); + String ipInYouId = split[0]; + Integer ipInYouIdCount = Integer.valueOf(split[1]); + Line line = new Line(ipInYouIdCount, ipInYouId); + lineList.add(line); + bufferedReaders.add(reader); + } - for (Map.Entry lineBufferedReaderEntry : currentTopRecordsFromBuffers.entrySet()) { - LOGGER.info(lineBufferedReaderEntry.getKey().getNumberOfIps() + " : " + lineBufferedReaderEntry.getKey().getNameOfIp()); + for (Line line : lineList) { + LOGGER.info(line.getNumberOfIps() + " : " + line.getNameOfIp()); } Map resultMap = new HashMap(NUMBER_OF_TOP_ELEMENTS_TO_PRINT_IN_HDFS); while (true) { - Map.Entry lineBufferedReaderEntry = currentTopRecordsFromBuffers.firstEntry(); - Integer value = lineBufferedReaderEntry.getKey().getNumberOfIps(); - String name = lineBufferedReaderEntry.getKey().getNameOfIp(); + String minimumKeyValueInList = getMinimumKeyValueInList(lineList); + int value = 0; + for (int i = 0; i < lineList.size(); i++) { + Line line = lineList.get(i); + if (line == null || !line.getNameOfIp().equals(minimumKeyValueInList)) { + continue; + } - addNewValueToMap(resultMap, value, name); + value += line.getNumberOfIps(); + String s = bufferedReaders.get(i).readLine(); + if (s != null) { + String[] split = s.split("\\t"); + String ipInYouId = split[0]; + Integer ipInYouIdCount = Integer.valueOf(split[1]); + Line nextLine = new Line(ipInYouIdCount, ipInYouId); + lineList.set(i, nextLine); + } else { + lineList.set(i, null); + } + } - readNewLineFromFile(currentTopRecordsFromBuffers, lineBufferedReaderEntry); + addNewValueToMap(resultMap, value, minimumKeyValueInList); - if (currentTopRecordsFromBuffers.size() == 0) { + if (isNoMoreLinesToProcess(lineList)) { break; } } @@ -75,6 +100,34 @@ private void processFiles(Configuration configuration, FileSystem fileSystem, Re fileSystem.close(); } + private boolean isNoMoreLinesToProcess(List lines) { + for (Line line : lines) { + if (line != null) { + return false; + } + } + + return true; + } + + private String getMinimumKeyValueInList(List lines) { + String result = null; + for (Line line : lines) { + if (line == null) { + continue; + } + + String temp = line.getNameOfIp(); + if (result == null) { + result = temp; + } else if (temp.compareTo(result) < 0) { + result = temp; + } + } + + return result; + } + private void processFiesInSeparateThreads(Configuration configuration, FileSystem fileSystem, RemoteIterator locatedFileStatusRemoteIterator) throws IOException { ExecutorService service = Executors.newCachedThreadPool(); @@ -102,32 +155,6 @@ private List createBufferedReaders(FileSystem fileSystem, Remote return readers; } - private TreeMap createBufferedReaderToLineMap(List readers) throws IOException { - TreeMap currentTopRecordsFromBuffers = new TreeMap(); - for (BufferedReader reader : readers) { - String s = reader.readLine(); - putNewLineInBuffersMap(currentTopRecordsFromBuffers, reader, s); - } - return currentTopRecordsFromBuffers; - } - - private void readNewLineFromFile(TreeMap currentTopRecordsFromBuffers, Map.Entry lineBufferedReaderEntry) throws IOException { - BufferedReader reader = lineBufferedReaderEntry.getValue(); - currentTopRecordsFromBuffers.pollFirstEntry(); - String nextLine = reader.readLine(); - if (nextLine != null) { - putNewLineInBuffersMap(currentTopRecordsFromBuffers, reader, nextLine); - } - } - - private void putNewLineInBuffersMap(TreeMap currentTopRecordsFromBuffers, BufferedReader reader, String nextLine) { - String[] split = nextLine.split("\\t"); - String ipInYouId = split[0]; - Integer ipInYouIdCount = Integer.valueOf(split[1]); - Line line = new Line(ipInYouIdCount, ipInYouId); - currentTopRecordsFromBuffers.put(line, reader); - } - private void addNewValueToMap(Map resultMap, Integer value, String name) { Integer previousValue = resultMap.get(name); if (previousValue == null) { @@ -209,7 +236,12 @@ public String getNameOfIp() { */ @Override public int compareTo(Line o) { - return o.getNumberOfIps().compareTo(numberOfIps); + int nameComparedValue = nameOfIp.compareTo(o.getNameOfIp()); + if (nameComparedValue == 0) { + return numberOfIps.compareTo(o.getNumberOfIps()); + } + + return nameComparedValue; } } } diff --git a/Homework2/src/main/screens/OprimizedResult.png b/Homework2/src/main/screens/OprimizedResult.png new file mode 100644 index 0000000..40dc605 Binary files /dev/null and b/Homework2/src/main/screens/OprimizedResult.png differ diff --git a/Homework2/src/main/screens/OprimizedResult_blockLists_150K_records_per_block.png b/Homework2/src/main/screens/OprimizedResult_blockLists_150K_records_per_block.png new file mode 100644 index 0000000..47623fe Binary files /dev/null and b/Homework2/src/main/screens/OprimizedResult_blockLists_150K_records_per_block.png differ diff --git a/Homework3/pom.xml b/Homework3/pom.xml index 6dcb589..8bbf357 100644 --- a/Homework3/pom.xml +++ b/Homework3/pom.xml @@ -180,7 +180,7 @@ host="${remote.host.ip}" username="${remote.host.name}" password="${remote.host.password}" - command="chmod -R 777 /root" + command="chmod -R 777 /root/${job.executor.script.name}" timeout="120000" usepty="true" /> diff --git a/Homework4/pom.xml b/Homework4/pom.xml index 453ec5c..6c743ee 100644 --- a/Homework4/pom.xml +++ b/Homework4/pom.xml @@ -184,7 +184,7 @@ host="${remote.host.ip}" username="${remote.host.name}" password="${remote.host.password}" - command="chmod -R 777 /root" + command="chmod -R 777 /root/${job.executor.script.name}" timeout="120000" usepty="true" /> diff --git a/Homework5/gs-yarn-basic-appmaster/pom.xml b/Homework5/gs-yarn-basic-appmaster/pom.xml index 48a1104..a5beb1e 100644 --- a/Homework5/gs-yarn-basic-appmaster/pom.xml +++ b/Homework5/gs-yarn-basic-appmaster/pom.xml @@ -13,13 +13,6 @@ Homework5 - - - - - - - diff --git a/Homework5/gs-yarn-basic-appmaster/src/main/java/hello/appmaster/AppmasterApplication.java b/Homework5/gs-yarn-basic-appmaster/src/main/java/hello/appmaster/AppmasterApplication.java index 9e76eb5..985a005 100644 --- a/Homework5/gs-yarn-basic-appmaster/src/main/java/hello/appmaster/AppmasterApplication.java +++ b/Homework5/gs-yarn-basic-appmaster/src/main/java/hello/appmaster/AppmasterApplication.java @@ -7,6 +7,7 @@ public class AppmasterApplication { public static void main(String[] args) { +// TODO place here web server SpringApplication.run(AppmasterApplication.class, args); } diff --git a/Homework5/gs-yarn-basic-client/pom.xml b/Homework5/gs-yarn-basic-client/pom.xml index f8622dd..ca19828 100644 --- a/Homework5/gs-yarn-basic-client/pom.xml +++ b/Homework5/gs-yarn-basic-client/pom.xml @@ -13,13 +13,6 @@ Homework5 - - - - - - - diff --git a/Homework5/gs-yarn-basic-container/pom.xml b/Homework5/gs-yarn-basic-container/pom.xml index b9f8c6d..82079ad 100644 --- a/Homework5/gs-yarn-basic-container/pom.xml +++ b/Homework5/gs-yarn-basic-container/pom.xml @@ -13,13 +13,6 @@ Homework5 - - - - - - - diff --git a/Homework5/gs-yarn-basic-dist/execute-job.sh b/Homework5/gs-yarn-basic-dist/execute-job.sh index 1d95342..03a19b1 100644 --- a/Homework5/gs-yarn-basic-dist/execute-job.sh +++ b/Homework5/gs-yarn-basic-dist/execute-job.sh @@ -13,8 +13,8 @@ bin/hdfs dfsadmin -safemode leave echo "NameNode leaved SafeMode state" -echo "Port 9000" >> /etc/ssh/sshd_config -service sshd restart +#echo "Port 9000" >> /etc/ssh/sshd_config +#service sshd restart #cd $HADOOP_PREFIX/etc/hadoop #sed 's/.*/hdfs:\/\/localhost:9000<\/value>/' core-site.xml.template > core-site.xml diff --git a/Homework5/gs-yarn-basic-dist/pom.xml b/Homework5/gs-yarn-basic-dist/pom.xml index 37187b8..e2570ae 100644 --- a/Homework5/gs-yarn-basic-dist/pom.xml +++ b/Homework5/gs-yarn-basic-dist/pom.xml @@ -123,7 +123,7 @@ host="${remote.host.ip}" username="${remote.host.name}" password="${remote.host.password}" - command="chmod -R 777 /root" + command="chmod -R 777 /root/${job.executor.script.name}" timeout="120000" usepty="true" /> diff --git a/Homework5/pom.xml b/Homework5/pom.xml index da32795..acfe8bd 100644 --- a/Homework5/pom.xml +++ b/Homework5/pom.xml @@ -7,7 +7,7 @@ org.springframework.boot spring-boot-starter-parent - 1.2.3.RELEASE + 1.3.6.RELEASE ru.hokan @@ -31,9 +31,6 @@ - - - 10.16.10.109 root 12345678