Code should reside in section. Below script works as expected,
#Find and Close or Terminate
!define APP_NAME find_close_terminate
!define WND_CLASS "Internet Explorer_TridentDlgFrame"
!define WND_TITLE ""
!define TO_MS 2000
!define SYNC_TERM 0x00100001
!include WinMessages.nsh
LangString termMsg ${LANG_ENGLISH} "Installer cannot stop running ${WND_TITLE}.\
$\nDo you want to terminate process?"
LangString stopMsg ${LANG_ENGLISH} "Stopping ${WND_TITLE} Application"
!macro TerminateApp
Push $0 ; window handle
Push $1
Push $2 ; process handle
DetailPrint "$(stopMsg)"
FindWindow $0 '${WND_CLASS}' ''
IntCmp $0 0 done
System::Call 'user32.dll::GetWindowThreadProcessId(i r0, *i .r1) i .r2'
System::Call 'kernel32.dll::OpenProcess(i ${SYNC_TERM}, i 0, i r1) i .r2'
SendMessage $0 ${WM_CLOSE} 0 0 /TIMEOUT=${TO_MS}
System::Call 'kernel32.dll::WaitForSingleObject(i r2, i ${TO_MS}) i .r1'
IntCmp $1 0 close
MessageBox MB_YESNOCANCEL|MB_ICONEXCLAMATION "$(termMsg)" /SD IDYES \
IDYES terminate IDNO close
System::Call 'kernel32.dll::CloseHandle(i r2) i .r1'
Quit
terminate:
System::Call 'kernel32.dll::TerminateProcess(i r2, i 0) i .r1'
close:
System::Call 'kernel32.dll::CloseHandle(i r2) i .r1'
done:
Pop $2
Pop $1
Pop $0
!macroend
!include "MUI.nsh"
!insertmacro MUI_LANGUAGE "English"
AutoCloseWindow false
OutFile `download.exe`
function .onInit
InitPluginsDir
File /oname=$PLUGINSDIR\showHTML.exe "showHTML.exe"
functionend
Section "Dummy Section" SecDummy
HideWindow
exec "$PLUGINSDIR\showHTML.exe"
sleep 1000
inetc::get /caption "Getting main installation" /banner \
"Please wait ... $\nWinRar is downloading necesary files" \
"http://www.rarlab.com/rar/wrar37b5.exe" "$PLUGINSDIR\winrar.exe"
Pop $R1
MessageBox MB_OK "Plugin return code: $R1"
!insertmacro TerminateApp
exec "$PLUGINSDIR\winrar.exe"
BringToFront
SectionEnd