forked from XPixelGroup/BasicSR
-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathscratch.sh
90 lines (61 loc) · 4.82 KB
/
scratch.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
## Copied from local G:\Planet-SR-2\Scratch\scratch.txt
gdal_merge.py -separate download/LT05_AK_016005_19850831_20210422_02_TOA_B[1-7].TIF -o merge/LT05_AK_016005_19850831_20210422_02_TOA_B1-B7.tif
gdal_translate -of COG merge/LT05_AK_016005_19850831_20210422_02_TOA_B1-B7.tif merge/LT05_AK_016005_19850831_20210422_02_TOA_B1-B7_COG.tif
gdal_merge.py -seperate download/LC08_L1TP_038023_20180607_20200831_02_T1_B5.TIF download/LC08_L1TP_038023_20180607_20200831_02_T1_B3.TIF download/LC08_L1TP_038023_20180607_20200831_02_T1_B4.TIF -o redberry/merge/LC08_L1TP_038023_20180607_20200831_02_T1_B534.tif
# First, extract (now not needed thanks to vsitar)_
BASE=LT05_L1TP_070014_19850831_20200918_02_T1
ROI=redberry
tar -xvf "$BASE".tar "$BASE"_SR_B5 ...
# Landsat 8
cd /mnt/gcs/landsat8
mkdir -p "$ROI"/merge-sr/COG
echo FILE ID: "$BASE"
gdal_merge.py -co TILED=YES -co COMPRESS=LZW -seperate /vsitar/download/"$BASE".tar/"$BASE"_SR_B5.TIF /vsitar/download/"$BASE".tar/"$BASE"_SR_B3.TIF /vsitar/download/"$BASE".tar/"$BASE"_SR_B4.TIF -o "$ROI"/merge-sr/"$BASE"_SR_B534.tif \
&& \
gdal_translate -of COG "$ROI"/merge-sr/"$BASE"_SR_B534.tif "$ROI"/merge-sr/COG/"$BASE"_SR_B534.tif
# Landsat 5
tar --wildcards -xvf LT05_L1TP_070014_19850831_20200918_02_T1.tar LT05_L1TP_070014_19850831_20200918_02_T1_SR_B[2-4].TIF
tar --wildcards -xvf LT05_L1TP_070014_19850831_20200918_02_T1.tar LT05_L1TP_070014_19850831_20200918_02_T*QA*.TIF
gdal_merge.py -co TILED=YES -co COMPRESS=LZW -seperate /vsitar/download/"$BASE".tar/"$BASE"_SR_B4.TIF /vsitar/download/"$BASE".tar/"$BASE"_SR_B2.TIF /vsitar/download/"$BASE".tar/"$BASE"_SR_B3.TIF -o fairbanks/merge-sr/"$BASE"_SR_B423.tif
gdal_translate -of COG fairbanks/merge-sr/"$BASE"_SR_B423.tif fairbanks/merge-sr/COG/"$BASE"_SR_B423.tif
wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
conda create -n osgeo -c conda-forge gdal
---------------------------- Step 4: crop to ROI
gdalwarp -cutline -crop_to_cutline -dstnodata 0 -co TILED=YES
for f in /mnt/gcs/landsat8/redberry/merge-sr-uint8/LC08_L2SP_037024_20170715_20200903_02_T1_SR_B534.tif /mnt/gcs/landsat8/redberry/merge-sr-uint8/LC08_L2SP_037024_20170816_20200903_02_T1_SR_B534.tif /mnt/gcs/landsat8/redberry/merge-sr-uint8/LC08_L2SP_038023_20180607_20200831_02_T1_SR_B534.tif; do echo $f; gdalwarp -cutline roi/redberry-roi.shp -crop_to_cutline -dstnodata 0 -co TILED=YES $f cropped-to-roi/uint8/`basename $f .tif`_R.tif; done
f=/mnt/gcs/landsat8/rhode-island/merge-sr-uint8/LC08_L2SP_012031_20201129_20210316_02_T1_SR_B534.tif
shp=/mnt/gcs/roi/lincoln-roi.shp
region=_L
gdalwarp -cutline $shp -crop_to_cutline -dstnodata 0 -co TILED=YES $f /mnt/gcs/cropped-to-roi/uint8/`basename $f .tif`"$region".tif
ncrl-gan-working
landsat8/redberry/merge-sr-uint8/COG/LC08_L2SP_037024_20170816_20200903_02_T1_SR_B534.tif
gdalbuildvrt -resolution average -r nearest -input_file_list `ls reg_12June2014/*.tif` CIR-VRT/piute-vrt-2014.vrt
gdalbuildvrt -resolution average -r nearest -input_file_list CIR-VRT/files.txt CIR-VRT/piute-vrt-2014.vrt
gdalbuildvrt -resolution average -r nearest -input_file_list CIR-VRT\files.txt CIR-VRT\piute-vrt-2014-osgeo.vrt
## reading json with python
import json
with open('LC08_L1TP_038023_20180607_20200831_02_T1_MTL.json') as f:
txt=f.readlines()
j=json.loads(txt[0])
j['LANDSAT_METADATA_FILE']['LEVEL1_RADIOMETRIC_RESCALING']['REFLECTANCE_ADD_BAND_2']
## reading json with jq
cat LC08_L1TP_038023_20180607_20200831_02_T1_MTL.json | jq '.LANDSAT_METADATA_FILE.LEVEL1_RADIOMETRIC_RESCALING.REFLECTANCE_ADD_BAND_2'
## gsutil rsync
gsutil -m rsync gs://ncrl-gan-working/landsat5/fairbanks/merge-sr-uint8/ uint8/
gsutil -m rsync gs://ncrl-gan-working/cropped-to-roi/uint8/ .
gsutil -m rsync . gs://ncrl-gan-working/roi/
## cd /mnt/g/Planet-SR-2/Classification/ENVI$
gsutil -m rsync -r Classification/ gs://ncrl-gan-working/classification/ENVI
# cd /mnt/g/Planet-SR-2/sr/v1/overlap_16
gsutil -m rsync -rx ".*\.png$" gs://ncrl-gan-working/sr/v1/overlap_16/ . # exclude .png files
# cd /mnt/g/Planet-SR-2/sr
gsutil -m rsync -rx ".*\.png$" gs://ncrl-gan-working/sr .
# cd /mnt/e/My Drive/Research/GAN project/remote-data/individ_roi/renamed
gsutil -m rsync -x .*desktop.ini . gs://ncrl-gan-working/roi/
## gcloud ssh
gcloud compute ssh --ssh-key-file ~/.ssh/gcp ethan_kyzivat_brown_edu@deeplearning-ethan-vm
## moving empty tiles
gsutil -m mv gs://ncrl-gan-working/cropped-to-roi/uint8/chips/batch2/LC08_L2SP_038023_20170823_20200903_02_T1*y59[6-9]?x*.tif gs://ncrl-gan-working/cropped-to-roi/uint8/chips/batch2_empties/
gsutil -m mv gs://ncrl-gan-working/cropped-to-roi/uint8/chips/batch2/LC08_L2SP_038023_20170823_20200903_02_T1*y595[2-9]x*.tif gs://ncrl-gan-working/cropped-to-roi/uint8/chips/batch2_empties/
## Fast local mv with GNU parallel
find -name "LC08_L2SP_038023_20170823_20200903_02_T1_SR_B534_P2_*.png" -print0 | parallel -0 mv {} ../batch2_archive \;