python
/
incremental_training
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Update

pull/1/head
Mike Kraus 2019-11-05 21:05:14 +01:00
parent bf858f6bf3
commit cc33e2a183
3 changed files with 95 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

167
.idea/workspace.xml
View File

@ -1,7 +1,11 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ChangeListManager">
<list default="true" id="f3f89d3d-fe06-4be5-b478-61463c4bc31a" name="Default Changelist" comment="" />
<list default="true" id="f3f89d3d-fe06-4be5-b478-61463c4bc31a" name="Default Changelist" comment="">
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/dags/initial_model_DAG.py" beforeDir="false" afterPath="$PROJECT_DIR$/dags/initial_model_DAG.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/dags/update_DAG.py" beforeDir="false" afterPath="$PROJECT_DIR$/dags/update_DAG.py" afterDir="false" />
</list>
<option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
@ -13,8 +17,8 @@
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/dags/src/data/kafka_producer.py">
<provider selected="true" editor-type-id="text-editor">
<state>
<caret selection-end-line="37" />
<state relative-caret-position="120">
<caret line="8" lean-forward="true" selection-end-line="37" />
<folding>
<element signature="e#0#32#0" expanded="true" />
</folding>
@ -22,11 +26,32 @@
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/dags/src/models/initial_model_functions.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="60">
<caret line="4" column="45" lean-forward="true" selection-start-line="4" selection-start-column="45" selection-end-line="4" selection-end-column="45" />
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/dags/initial_model_DAG.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="135">
<caret line="13" column="10" selection-start-line="13" selection-start-column="10" selection-end-line="13" selection-end-column="10" />
<folding>
<element signature="e#1#15#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/dags/stream_DAG.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="435">
<caret line="29" column="1" selection-start-line="1" selection-end-line="29" selection-end-column="1" />
<state relative-caret-position="115">
<caret line="17" column="24" lean-forward="true" selection-end-line="32" />
<folding>
<element signature="e#1#15#0" expanded="true" />
</folding>
@ -37,20 +62,21 @@
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/dags/src/models/update_functions.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="255">
<caret line="17" column="27" selection-end-line="114" />
<state relative-caret-position="120">
<caret line="8" column="43" selection-start-line="8" selection-start-column="43" selection-end-line="8" selection-end-column="43" />
<folding>
<element signature="e#0#12#0" expanded="true" />
<element signature="e#166#368#0" />
</folding>
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="true">
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/.gitignore">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="30">
<caret line="2" column="7" lean-forward="true" selection-start-line="2" selection-start-column="7" selection-end-line="2" selection-end-column="7" />
<caret line="2" column="7" selection-start-line="2" selection-start-column="7" selection-end-line="2" selection-end-column="7" />
</state>
</provider>
</entry>
@ -63,8 +89,8 @@
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/dags/src/data/data_functions.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="255">
<caret line="17" selection-end-line="81" />
<state relative-caret-position="174">
<caret line="74" column="7" selection-start-line="74" selection-start-column="7" selection-end-line="74" selection-end-column="7" />
<folding>
<element signature="e#0#47#0" expanded="true" />
</folding>
@ -72,11 +98,11 @@
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<file pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/dags/update_DAG.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="915">
<caret line="61" column="17" lean-forward="true" selection-start-line="61" selection-start-column="17" selection-end-line="61" selection-end-column="17" />
<state relative-caret-position="233">
<caret line="24" column="10" selection-start-line="24" selection-start-column="10" selection-end-line="24" selection-end-column="10" />
<folding>
<element signature="e#1#15#0" expanded="true" />
</folding>
@ -107,6 +133,7 @@
<find>.p</find>
<find>load_base</find>
<find>retrain</find>
<find>keras</find>
</findStrings>
<replaceStrings>
<replace>NUM_CLASSES</replace>
@ -157,24 +184,24 @@
<option value="$PROJECT_DIR$/Kubernetes/serve/deploy_minikube.yaml" />
<option value="$PROJECT_DIR$/dags/initial_model_dag.py" />
<option value="$PROJECT_DIR$/dags/stream_DAG.py" />
<option value="$PROJECT_DIR$/dags/initial_model_DAG.py" />
<option value="$PROJECT_DIR$/docker-compose-project.yml" />
<option value="$PROJECT_DIR$/dags/src/data/data_functions.py" />
<option value="$PROJECT_DIR$/dags/src/models/initial_model_functions.py" />
<option value="$PROJECT_DIR$/dags/src/models/retrain_functions.py" />
<option value="$PROJECT_DIR$/dags/src/preprocessing/preprocessing_functions.py" />
<option value="$PROJECT_DIR$/dags/update_DAG.py" />
<option value="$PROJECT_DIR$/dags/src/models/update_functions.py" />
<option value="$PROJECT_DIR$/dags/initial_model_DAG.py" />
<option value="$PROJECT_DIR$/dags/update_DAG.py" />
<option value="$PROJECT_DIR$/dags/src/data/kafka_producer.py" />
<option value="$PROJECT_DIR$/.gitignore" />
<option value="$PROJECT_DIR$/dags/initial_model_DAG.py" />
<option value="$PROJECT_DIR$/dags/initial_model_DAG.py" />
<option value="$PROJECT_DIR$/dags/update_DAG.py" />
<option value="$PROJECT_DIR$/dags/update_DAG.py" />
</list>
</option>
</component>
<component name="ProjectFrameBounds">
<option name="x" value="398" />
<option name="y" value="-962" />
<component name="ProjectFrameBounds" extendedState="1">
<option name="x" value="158" />
<option name="y" value="88" />
<option name="width" value="1454" />
<option name="height" value="934" />
</component>
@ -183,6 +210,7 @@
<foldersAlwaysOnTop value="true" />
</navigator>
<panes>
<pane id="Scope" />
<pane id="ProjectPane">
<subPane>
<expand>
@ -190,6 +218,11 @@
<item name="incremental_training" type="b2602c69:ProjectViewProjectNode" />
<item name="incremental_training" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="incremental_training" type="b2602c69:ProjectViewProjectNode" />
<item name="incremental_training" type="462c0819:PsiDirectoryNode" />
<item name="airflow_docker" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="incremental_training" type="b2602c69:ProjectViewProjectNode" />
<item name="incremental_training" type="462c0819:PsiDirectoryNode" />
@ -215,22 +248,10 @@
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="models" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="incremental_training" type="b2602c69:ProjectViewProjectNode" />
<item name="incremental_training" type="462c0819:PsiDirectoryNode" />
<item name="data" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="incremental_training" type="b2602c69:ProjectViewProjectNode" />
<item name="incremental_training" type="462c0819:PsiDirectoryNode" />
<item name="data" type="462c0819:PsiDirectoryNode" />
<item name="used_for_model_update" type="462c0819:PsiDirectoryNode" />
</path>
</expand>
<select />
</subPane>
</pane>
<pane id="Scope" />
</panes>
</component>
<component name="PropertiesComponent">
@ -404,7 +425,7 @@
</todo-panel>
</component>
<component name="ToolWindowManager">
<frame x="398" y="-962" width="1454" height="934" extended-state="0" />
<frame x="158" y="88" width="1454" height="934" extended-state="0" />
<editor active="true" />
<layout>
<window_info active="true" content_ui="combo" id="Project" order="0" visible="true" weight="0.25610608" />
@ -495,16 +516,6 @@
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/dags/src/models/initial_model_functions.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="280">
<caret line="122" lean-forward="true" selection-end-line="122" />
<folding>
<element signature="e#1#13#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/docker-compose-project.yml">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="94">
@ -512,10 +523,31 @@
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/dags/initial_model_DAG.py">
<entry file="file://$PROJECT_DIR$/.gitignore">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="195">
<caret line="13" column="10" lean-forward="true" selection-start-line="13" selection-start-column="10" selection-end-line="13" selection-end-column="10" />
<state relative-caret-position="30">
<caret line="2" column="7" selection-start-line="2" selection-start-column="7" selection-end-line="2" selection-end-column="7" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/.gitattributes">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/dags/src/models/update_functions.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="120">
<caret line="8" column="43" selection-start-line="8" selection-start-column="43" selection-end-line="8" selection-end-column="43" />
<folding>
<element signature="e#0#12#0" expanded="true" />
<element signature="e#166#368#0" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/dags/stream_DAG.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="115">
<caret line="17" column="24" lean-forward="true" selection-end-line="32" />
<folding>
<element signature="e#1#15#0" expanded="true" />
</folding>
@ -524,63 +556,50 @@
</entry>
<entry file="file://$PROJECT_DIR$/dags/src/data/kafka_producer.py">
<provider selected="true" editor-type-id="text-editor">
<state>
<caret selection-end-line="37" />
<state relative-caret-position="120">
<caret line="8" lean-forward="true" selection-end-line="37" />
<folding>
<element signature="e#0#32#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/dags/stream_DAG.py">
<entry file="file://$PROJECT_DIR$/dags/src/models/initial_model_functions.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="435">
<caret line="29" column="1" selection-start-line="1" selection-end-line="29" selection-end-column="1" />
<folding>
<element signature="e#1#15#0" expanded="true" />
</folding>
<state relative-caret-position="60">
<caret line="4" column="45" lean-forward="true" selection-start-line="4" selection-start-column="45" selection-end-line="4" selection-end-column="45" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/dags/src/data/data_functions.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="255">
<caret line="17" selection-end-line="81" />
<state relative-caret-position="174">
<caret line="74" column="7" selection-start-line="74" selection-start-column="7" selection-end-line="74" selection-end-column="7" />
<folding>
<element signature="e#0#47#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/dags/update_DAG.py">
<entry file="file://$PROJECT_DIR$/dags/initial_model_DAG.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="915">
<caret line="61" column="17" lean-forward="true" selection-start-line="61" selection-start-column="17" selection-end-line="61" selection-end-column="17" />
<state relative-caret-position="135">
<caret line="13" column="10" selection-start-line="13" selection-start-column="10" selection-end-line="13" selection-end-column="10" />
<folding>
<element signature="e#1#15#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/dags/src/models/update_functions.py">
<entry file="file://$PROJECT_DIR$/dags/update_DAG.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="255">
<caret line="17" column="27" selection-end-line="114" />
<state relative-caret-position="233">
<caret line="24" column="10" selection-start-line="24" selection-start-column="10" selection-end-line="24" selection-end-column="10" />
<folding>
<element signature="e#0#12#0" expanded="true" />
<element signature="e#1#15#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/.gitattributes">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/.gitignore">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="30">
<caret line="2" column="7" lean-forward="true" selection-start-line="2" selection-start-column="7" selection-end-line="2" selection-end-column="7" />
</state>
</provider>
</entry>
</component>
</project>

2
dags/initial_model_DAG.py
View File

@ -11,7 +11,7 @@ INITIAL_MODEL_PATH = "/models/current_model/initial_model.H5"
BATCH_SIZE = 128
NUM_CLASSES = 10
EPOCHS = 1
EPOCHS = 4
args = {
'owner': 'airflow',

2
dags/update_DAG.py
View File

@ -22,7 +22,7 @@ PATH_MODEL_ARCHIVE = '/models/archive/'
BATCH_SIZE = 128
NUM_CLASSES = 10
EPOCHS = 1
EPOCHS = 4
args = {