Python plugin jinja template #4858

joseph-robertson · 2023-04-07T22:03:59Z

Pull request overview

Addresses Approach 1(A), but using a python measure: #4634 (comment)

I've created two measures:

PythonPluginRuby (uses python_plugin_program_erb.py template)
PythonPluginPython (uses python_plugin_program_jinja.py template)

Both measures are called by compact_python_plugin.osw.

$ /c/openstudio-3.6.0-alpha/bin/openstudio.exe labs run -w resources/Examples/compact_osw/compact_python_plugin.osw

Appears to be working successfully.

Update: I've removed sys.path.append and tried running with OS v3.7.0 (w/out labs), and it appears to be working successfully.

Pull Request Author

Model API Changes / Additions
Any new or modified fields have been implemented in the EnergyPlus ForwardTranslator (and ReverseTranslator as appropriate)
Model API methods are tested (in src/model/test)
EnergyPlus ForwardTranslator Tests (in src/energyplus/Test)
If a new object or method, added a test in NREL/OpenStudio-resources: Add Link
If needed, added VersionTranslation rules for the objects (src/osversion/VersionTranslator.cpp)
Verified that C# bindings built fine on Windows, partial classes used as needed, etc.
All new and existing tests passes
If methods have been deprecated, update rest of code to use the new methods

Labels:

If change to an IDD file, add the label IDDChange
If breaking existing API, add the label APIChange
If deemed ready, add label Pull Request - Ready for CI so that CI builds your PR

Review Checklist

This will not be exhaustively relevant to every PR.

Perform a Code Review on GitHub
Code Style, strip trailing whitespace, etc.
All related changes have been implemented: model changes, model tests, FT changes, FT tests, VersionTranslation, OS App
Labeling is ok
If defect, verify by running develop branch and reproducing defect, then running PR and reproducing fix
If feature, test running new feature, try creative ways to break it
CI status: all green or justified

resources/Examples/compact_osw/files/python_plugin_program_erb.py

jmarrec · 2023-04-25T07:47:28Z

resources/Examples/compact_osw/files/python_plugin_program_erb.py

+from pyenergyplus.plugin import EnergyPlusPlugin
+
+import os, sys
+sys.path.append('C:/Python38/Lib/site-packages') # this should (needs to?) be same version as E+'s version


We're going to have to make this path to system python's site_packages some kind of cmake replacement or a command line arg if we want to test this.

But more broadly, this is going to pose the question of whether we should/need to provide some common python packages out of the box like we do for ruby gems (and take into account the pain that it is to maintain that stuff). I'm specifically thinking about native packages (related to this issue #4846), (pandas does have a C extension as well, I think something about numpy may be special)

@kbenne @tijcolem

Hi @tijcolem, do you have any guidance here? Do we want to make running this OSW into an actual test? Or should these changes (i.e., demonstrating using Python Plugin EMS through both Ruby and Python measures) just be examples that we could point users to?

@jmarrec

Following #4868 you can make your test pass the --python_home or --python_path and it'll work (provided the machine it runs on does have jinja2 installed)

I think we should discuss whether we should always add some common python packages like Jinja2 (#4906 (comment))

@jmarrec @kbenne

Has there been any more discussion/movement on #4906?

Assuming the jinja2 package doesn't get bundled into the CLI, what paths would be set for --python_home and --python_path?

@jmarrec

See here:

OpenStudio/src/cli/CMakeLists.txt

Lines 343 to 393 in 0ab0280

if(NOT EXISTS "${Python_ROOT_DIR}")

# Python_STDLIB: we expect it to be

# Unix: ~/.pyenv/versions/3.8.12/lib/python3.8 or

# or on CI: /usr/lib/python3.8/ ... with numpy if install via pip3 and not apt install python3-numpy in `/usr/local/lib/python3.8/dist-packages/`

# Windows C:\Python38\Lib

cmake_path(GET Python_STDLIB PARENT_PATH Python_ROOT_DIR)

if(UNIX)

cmake_path(GET Python_ROOT_DIR PARENT_PATH Python_ROOT_DIR)

endif()

endif()

if(UNIX)

if(EXISTS "${Python_SITELIB}")

set(PYTHON_PATH "${Python_SITELIB}" "${Python_STDLIB}/lib-dynload")

else()

set(PYTHON_PATH "${Python_STDLIB}/lib-dynload")

endif()

if(NOT APPLE)

set(EXTRA_LOCAL_DIST "/usr/local/lib/python3.8/dist-packages")

if (EXISTS "${EXTRA_LOCAL_DIST}")

list(APPEND PYTHON_PATH "${EXTRA_LOCAL_DIST}")

endif()

endif()

else()

set(PYTHON_PATH "$<SHELL_PATH:${Python_SITELIB}>")

endif()

message(DEBUG "PYTHON_PATH=${PYTHON_PATH}")

add_test(NAME OpenStudioCLI.Labs.execute_python_script.numpy.explicit_sys_path_insert

COMMAND $<TARGET_FILE:openstudio> labs execute_python_script execute_python_script_with_numpy.py ${Python_STDLIB}

WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/test/"

)

add_test(NAME OpenStudioCLI.Labs.execute_python_script.numpy.python_path

COMMAND $<TARGET_FILE:openstudio> labs

"$<$<BOOL:${PYTHON_PATH}>:--python_path;$<JOIN:${PYTHON_PATH},;--python_path;>>"

execute_python_script execute_python_script_with_numpy.py

COMMAND_EXPAND_LISTS

WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/test/"

)

add_test(NAME OpenStudioCLI.Labs.execute_python_script.numpy.python_home

COMMAND $<TARGET_FILE:openstudio> labs

--python_home "$<SHELL_PATH:${Python_ROOT_DIR}>"

execute_python_script execute_python_script_with_numpy.py

WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/test/"

)

else()

message(AUTHOR_WARNING "Cannot run the python numpy test, as numpy isn't installed on your system python")

(py39)julien@OS-build-release$ Products/openstudio labs -c "import jinja2" Traceback (most recent call last): File "<string>", line 1, in <module> ModuleNotFoundError: No module named 'jinja2' terminate called after throwing an instance of 'std::runtime_error' what(): Error executing Python code Aborted (core dumped) (py39)julien@OS-build-release$ Products/openstudio labs --python_path /home/julien/.pyenv/versions/3.8.13/lib/python3.8/site-packages -c "import jinja2"

…to plugin-jinja-template

joseph-robertson · 2024-02-15T18:23:38Z

@jmarrec This appears to be working now (i.e., can import pandas and jinja2). How can we formalize running compact_python_plugin.osw as a test? Add something to src/cli/CMakeLists?

jmarrec · 2024-02-16T08:18:47Z

resources/Examples/compact_osw/measures/PythonPluginPython/measure.py

@@ -0,0 +1,132 @@
+import os, sys


Suggested change

import os, sys

import os

import sys

Generally speaking you should pip install black isort on your system and reformat your python files via

isort . black -l 120 .

jmarrec · 2024-02-16T08:21:29Z

resources/Examples/compact_osw/compact_python_plugin.osw

Move this to resources/workflow.

Or it'll get installed: resources/Examples is installed to disk:

OpenStudio/resources/CMakeLists.txt

Line 420 in d8bbf52

install(DIRECTORY "Examples" DESTINATION . COMPONENT "CLI")

jmarrec · 2024-02-16T08:29:45Z

resources/Examples/compact_osw/files/python_plugin_program_erb.py

This should be inside the measures directory inside resources/ , so it is a proper measure and can be moved around.

…B/Jinja templates are python files

… pathlib.Path (modernize python code)

… to ensure plugins do run

jmarrec · 2024-02-16T09:52:50Z

@jmarrec This appears to be working now (i.e., can import pandas and jinja2). How can we formalize running compact_python_plugin.osw as a test? Add something to src/cli/CMakeLists?

Yes. like you've done. But it bothers me we don't check at all if E+ actually runs the python plugins

jmarrec · 2024-02-16T09:53:35Z

@joseph-robertson I've made some adjustments (moved things around, linted) + actually tested the plugins are running.

Assuming CI comes clean and you're ok with the changes, this can drop.

ci-commercialbuildings · 2024-02-16T10:08:44Z

CI Results for 148f089:

Ubuntu 20.04:
Ubuntu 22.04:
Darwin:
- OpenStudio-3.7.0+148f089e91-Darwin-x86_64.dmg
- OpenStudio-3.7.0+148f089e91-Darwin-x86_64.tar.gz
Windows:
- OpenStudio-3.7.0+148f089e91-Windows.exe
- OpenStudio-3.7.0+148f089e91-Windows.tar.gz
Code Coverage:

joseph-robertson added 3 commits April 7, 2023 15:00

Stub python equiv of python_plugin_2.

1c55207

Add osw pointing to new measure.

c72e04f

Add in plugin jinja template.

17e31fb

joseph-robertson self-assigned this Apr 7, 2023

joseph-robertson added 7 commits April 10, 2023 15:52

Remove argument from osw.

8a1b854

Update jinja template file.

3f3b0e6

Render jinja template using context.

bb2036a

Merge branch 'develop' into plugin-jinja-template

14809fc

Add the python erb template.

1548fb2

Add analogous ruby osw and measure.

b8a9ed5

Minor updates to python measure.

58b51d5

joseph-robertson mentioned this pull request Apr 17, 2023

ExternalFile issues with labs #4863

Closed

Update names of template files.

66c37d0

jmarrec reviewed Apr 25, 2023

View reviewed changes

joseph-robertson added 10 commits April 25, 2023 09:51

Stub python equiv of python_plugin_2.

5e368d9

Add osw pointing to new measure.

2ee386e

Add in plugin jinja template.

eca71d3

Remove argument from osw.

e25f65c

Update jinja template file.

c0dc768

Render jinja template using context.

9b8c1ea

Add the python erb template.

cf08d3a

Add analogous ruby osw and measure.

97e3b50

Minor updates to python measure.

5883844

Update names of template files.

2182b84

jmarrec force-pushed the plugin-jinja-template branch from 66c37d0 to 2182b84 Compare April 25, 2023 07:51

joseph-robertson added 2 commits April 25, 2023 10:05

Merge branch 'plugin-jinja-template' of github.com:NREL/OpenStudio in…

93b5cb2

…to plugin-jinja-template

Merge branch 'develop' into plugin-jinja-template

9ee6054

joseph-robertson added the Pull Request - Ready for CI This pull request if finalized and is ready for continuous integration verification prior to merge. label Apr 25, 2023

joseph-robertson added 2 commits April 25, 2023 14:41

Import on separate lines in template files.

d35e005

Condense ruby and python measures into a single osw.

117a9f6

Update measure xml files.

ec727d7

joseph-robertson linked an issue Apr 25, 2023 that may be closed by this pull request

Python plugin next steps #4634

Open

2 tasks

jmarrec mentioned this pull request Jul 4, 2023

Support Testing of Python Measures with standard OpenStudio install #4906

Closed

joseph-robertson added 3 commits August 30, 2023 10:02

Merge branch 'develop' into plugin-jinja-template

6b78ca1

Merge branch 'develop' into plugin-jinja-template

fbd81db

Remove sys.path.append and verify import of jinja2 and pandas.

42c1857

joseph-robertson marked this pull request as ready for review February 15, 2024 18:23

joseph-robertson added 2 commits February 15, 2024 15:12

Try adding src cli test to cmakelists.

fedec8b

Try that again.

dc3ce78

jmarrec reviewed Feb 16, 2024

View reviewed changes

jmarrec added 6 commits February 16, 2024 09:39

Move things around

6df7aa5

Lint and rename templates to that IDE / lint tools don't think the ER…

e4dcf07

…B/Jinja templates are python files

Use unique names for variables (both measures are run together) + use…

4ed9478

… pathlib.Path (modernize python code)

Make the Plugins print a message to stdout and test for that in CTest…

01c8ee4

… to ensure plugins do run

Shorten run period of example_model.osm for CLI tests so they run faster

4ad2e07

Shush test_with_analysis

148f089

joseph-robertson merged commit 02cf3b8 into develop Feb 16, 2024
4 of 6 checks passed

joseph-robertson deleted the plugin-jinja-template branch February 16, 2024 15:55

joseph-robertson mentioned this pull request Feb 23, 2024

Python plugin next steps #4634

Open

2 tasks

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Python plugin jinja template #4858

Python plugin jinja template #4858

	if(NOT EXISTS "${Python_ROOT_DIR}")
	# Python_STDLIB: we expect it to be
	# Unix: ~/.pyenv/versions/3.8.12/lib/python3.8 or
	# or on CI: /usr/lib/python3.8/ ... with numpy if install via pip3 and not apt install python3-numpy in `/usr/local/lib/python3.8/dist-packages/`
	# Windows C:\Python38\Lib
	cmake_path(GET Python_STDLIB PARENT_PATH Python_ROOT_DIR)
	if(UNIX)
	cmake_path(GET Python_ROOT_DIR PARENT_PATH Python_ROOT_DIR)
	endif()
	endif()

	if(UNIX)
	if(EXISTS "${Python_SITELIB}")
	set(PYTHON_PATH "${Python_SITELIB}" "${Python_STDLIB}/lib-dynload")
	else()
	set(PYTHON_PATH "${Python_STDLIB}/lib-dynload")
	endif()

	if(NOT APPLE)
	set(EXTRA_LOCAL_DIST "/usr/local/lib/python3.8/dist-packages")
	if (EXISTS "${EXTRA_LOCAL_DIST}")
	list(APPEND PYTHON_PATH "${EXTRA_LOCAL_DIST}")
	endif()
	endif()
	else()
	set(PYTHON_PATH "$<SHELL_PATH:${Python_SITELIB}>")
	endif()

	message(DEBUG "PYTHON_PATH=${PYTHON_PATH}")

	add_test(NAME OpenStudioCLI.Labs.execute_python_script.numpy.explicit_sys_path_insert
	COMMAND $<TARGET_FILE:openstudio> labs execute_python_script execute_python_script_with_numpy.py ${Python_STDLIB}
	WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/test/"
	)

	add_test(NAME OpenStudioCLI.Labs.execute_python_script.numpy.python_path
	COMMAND $<TARGET_FILE:openstudio> labs
	"$<$<BOOL:${PYTHON_PATH}>:--python_path;$<JOIN:${PYTHON_PATH},;--python_path;>>"
	execute_python_script execute_python_script_with_numpy.py
	COMMAND_EXPAND_LISTS
	WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/test/"
	)

	add_test(NAME OpenStudioCLI.Labs.execute_python_script.numpy.python_home
	COMMAND $<TARGET_FILE:openstudio> labs
	--python_home "$<SHELL_PATH:${Python_ROOT_DIR}>"
	execute_python_script execute_python_script_with_numpy.py
	WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/test/"
	)
	else()
	message(AUTHOR_WARNING "Cannot run the python numpy test, as numpy isn't installed on your system python")

Python plugin jinja template #4858

Python plugin jinja template #4858

Conversation

Pull request overview

Pull Request Author

Review Checklist

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment