aboutsummaryrefslogtreecommitdiffstats
path: root/waflib/extras/remote.py
blob: 3b038f772b544444c126f3af8151b0431c2f7539 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
#!/usr/bin/env python
# encoding: utf-8
# Remote Builds tool using rsync+ssh

__author__ = "Jérôme Carretero <cJ-waf@zougloub.eu>"
__copyright__ = "Jérôme Carretero, 2013"

"""
Simple Remote Builds
********************

This tool is an *experimental* tool (meaning, do not even try to pollute
the waf bug tracker with bugs in here, contact me directly) providing simple
remote builds.

It uses rsync and ssh to perform the remote builds.
It is intended for performing cross-compilation on platforms where
a cross-compiler is either unavailable (eg. MacOS, QNX) a specific product
does not exist (eg. Windows builds using Visual Studio) or simply not installed.
This tool sends the sources and the waf script to the remote host,
and commands the usual waf execution.

There are alternatives to using this tool, such as setting up shared folders,
logging on to remote machines, and building on the shared folders.
Electing one method or another depends on the size of the program.


Usage
=====

1. Set your wscript file so it includes a list of variants,
   e.g.::

     from waflib import Utils
     top = '.'
     out = 'build'

     variants = [
      'linux_64_debug',
      'linux_64_release',
      'linux_32_debug',
      'linux_32_release',
      ]

     from waflib.extras import remote

     def options(opt):
         # normal stuff from here on
         opt.load('compiler_c')

     def configure(conf):
         if not conf.variant:
             return
         # normal stuff from here on
         conf.load('compiler_c')

     def build(bld):
         if not bld.variant:
             return
         # normal stuff from here on
         bld(features='c cprogram', target='app', source='main.c')


2. Build the waf file, so it includes this tool, and put it in the current
   directory

   .. code:: bash

      ./waf-light --tools=remote

3. Set the host names to access the hosts:

   .. code:: bash

      export REMOTE_QNX=user@kiunix

4. Setup the ssh server and ssh keys

   The ssh key should not be protected by a password, or it will prompt for it every time.
   Create the key on the client:

   .. code:: bash

      ssh-keygen -t rsa -f foo.rsa

   Then copy foo.rsa.pub to the remote machine (user@kiunix:/home/user/.ssh/authorized_keys),
   and make sure the permissions are correct (chmod go-w ~ ~/.ssh ~/.ssh/authorized_keys)

   A separate key for the build processes can be set in the environment variable WAF_SSH_KEY.
   The tool will then use 'ssh-keyscan' to avoid prompting for remote hosts, so
   be warned to use this feature on internal networks only (MITM).

   .. code:: bash

      export WAF_SSH_KEY=~/foo.rsa

5. Perform the build:

   .. code:: bash

      waf configure_all build_all --remote

"""


import getpass, os, re, sys
from collections import OrderedDict
from waflib import Context, Options, Utils, ConfigSet

from waflib.Build import BuildContext, CleanContext, InstallContext, UninstallContext
from waflib.Configure import ConfigurationContext


is_remote = False
if '--remote' in sys.argv:
	is_remote = True
	sys.argv.remove('--remote')

class init(Context.Context):
	"""
	Generates the *_all commands
	"""
	cmd = 'init'
	fun = 'init'
	def execute(self):
		for x in list(Context.g_module.variants):
			self.make_variant(x)
		lst = ['remote']
		for k in Options.commands:
			if k.endswith('_all'):
				name = k.replace('_all', '')
				for x in Context.g_module.variants:
					lst.append('%s_%s' % (name, x))
			else:
				lst.append(k)
		del Options.commands[:]
		Options.commands += lst

	def make_variant(self, x):
		for y in (BuildContext, CleanContext, InstallContext, UninstallContext):
			name = y.__name__.replace('Context','').lower()
			class tmp(y):
				cmd = name + '_' + x
				fun = 'build'
				variant = x
		class tmp(ConfigurationContext):
			cmd = 'configure_' + x
			fun = 'configure'
			variant = x
			def __init__(self, **kw):
				ConfigurationContext.__init__(self, **kw)
				self.setenv(x)

class remote(BuildContext):
	cmd = 'remote'
	fun = 'build'

	def get_ssh_hosts(self):
		lst = []
		for v in Context.g_module.variants:
			self.env.HOST = self.login_to_host(self.variant_to_login(v))
			cmd = Utils.subst_vars('${SSH_KEYSCAN} -t rsa,ecdsa ${HOST}', self.env)
			out, err = self.cmd_and_log(cmd, output=Context.BOTH, quiet=Context.BOTH)
			lst.append(out.strip())
		return lst

	def setup_private_ssh_key(self):
		"""
		When WAF_SSH_KEY points to a private key, a .ssh directory will be created in the build directory
		Make sure that the ssh key does not prompt for a password
		"""
		key = os.environ.get('WAF_SSH_KEY', '')
		if not key:
			return
		if not os.path.isfile(key):
			self.fatal('Key in WAF_SSH_KEY must point to a valid file')
		self.ssh_dir = os.path.join(self.path.abspath(), 'build', '.ssh')
		self.ssh_hosts = os.path.join(self.ssh_dir, 'known_hosts')
		self.ssh_key = os.path.join(self.ssh_dir, os.path.basename(key))
		self.ssh_config = os.path.join(self.ssh_dir, 'config')
		for x in self.ssh_hosts, self.ssh_key, self.ssh_config:
			if not os.path.isfile(x):
				if not os.path.isdir(self.ssh_dir):
					os.makedirs(self.ssh_dir)
				Utils.writef(self.ssh_key, Utils.readf(key), 'wb')
				os.chmod(self.ssh_key, 448)

				Utils.writef(self.ssh_hosts, '\n'.join(self.get_ssh_hosts()))
				os.chmod(self.ssh_key, 448)

				Utils.writef(self.ssh_config, 'UserKnownHostsFile %s' % self.ssh_hosts, 'wb')
				os.chmod(self.ssh_config, 448)
		self.env.SSH_OPTS = ['-F', self.ssh_config, '-i', self.ssh_key]
		self.env.append_value('RSYNC_SEND_OPTS', '--exclude=build/.ssh')

	def skip_unbuildable_variant(self):
		# skip variants that cannot be built on this OS
		for k in Options.commands:
			a, _, b = k.partition('_')
			if b in Context.g_module.variants:
				c, _, _ = b.partition('_')
				if c != Utils.unversioned_sys_platform():
					Options.commands.remove(k)

	def login_to_host(self, login):
		return re.sub('(\w+@)', '', login)

	def variant_to_login(self, variant):
		"""linux_32_debug -> search env.LINUX_32 and then env.LINUX"""
		x = variant[:variant.rfind('_')]
		ret = os.environ.get('REMOTE_' + x.upper(), '')
		if not ret:
			x = x[:x.find('_')]
			ret = os.environ.get('REMOTE_' + x.upper(), '')
		if not ret:
			ret = '%s@localhost' % getpass.getuser()
		return ret

	def execute(self):
		global is_remote
		if not is_remote:
			self.skip_unbuildable_variant()
		else:
			BuildContext.execute(self)

	def restore(self):
		self.top_dir = os.path.abspath(Context.g_module.top)
		self.srcnode = self.root.find_node(self.top_dir)
		self.path = self.srcnode

		self.out_dir = os.path.join(self.top_dir, Context.g_module.out)
		self.bldnode = self.root.make_node(self.out_dir)
		self.bldnode.mkdir()

		self.env = ConfigSet.ConfigSet()

	def extract_groups_of_builds(self):
		"""Return a dict mapping each variants to the commands to build"""
		self.vgroups = {}
		for x in reversed(Options.commands):
			_, _, variant = x.partition('_')
			if variant in Context.g_module.variants:
				try:
					dct = self.vgroups[variant]
				except KeyError:
					dct = self.vgroups[variant] = OrderedDict()
				try:
					dct[variant].append(x)
				except KeyError:
					dct[variant] = [x]
				Options.commands.remove(x)

	def custom_options(self, login):
		try:
			return Context.g_module.host_options[login]
		except (AttributeError, KeyError):
			return {}

	def recurse(self, *k, **kw):
		self.env.RSYNC = getattr(Context.g_module, 'rsync', 'rsync -a --chmod=u+rwx')
		self.env.SSH = getattr(Context.g_module, 'ssh', 'ssh')
		self.env.SSH_KEYSCAN = getattr(Context.g_module, 'ssh_keyscan', 'ssh-keyscan')
		try:
			self.env.WAF = getattr(Context.g_module, 'waf')
		except AttributeError:
			try:
				os.stat('waf')
			except KeyError:
				self.fatal('Put a waf file in the directory (./waf-light --tools=remote)')
			else:
				self.env.WAF = './waf'

		self.extract_groups_of_builds()
		self.setup_private_ssh_key()
		for k, v in self.vgroups.items():
			task = self(rule=rsync_and_ssh, always=True)
			task.env.login = self.variant_to_login(k)

			task.env.commands = []
			for opt, value in v.items():
				task.env.commands += value
			task.env.variant = task.env.commands[0].partition('_')[2]
			for opt, value in self.custom_options(k):
				task.env[opt] = value
		self.jobs = len(self.vgroups)

	def make_mkdir_command(self, task):
		return Utils.subst_vars('${SSH} ${SSH_OPTS} ${login} "rm -fr ${remote_dir} && mkdir -p ${remote_dir}"', task.env)

	def make_send_command(self, task):
		return Utils.subst_vars('${RSYNC} ${RSYNC_SEND_OPTS} -e "${SSH} ${SSH_OPTS}" ${local_dir} ${login}:${remote_dir}', task.env)

	def make_exec_command(self, task):
		txt = '''${SSH} ${SSH_OPTS} ${login} "cd ${remote_dir} && ${WAF} ${commands}"'''
		return Utils.subst_vars(txt, task.env)

	def make_save_command(self, task):
		return Utils.subst_vars('${RSYNC} ${RSYNC_SAVE_OPTS} -e "${SSH} ${SSH_OPTS}" ${login}:${remote_dir_variant} ${build_dir}', task.env)

def rsync_and_ssh(task):

	# remove a warning
	task.uid_ = id(task)

	bld = task.generator.bld

	task.env.user, _, _ = task.env.login.partition('@')
	task.env.hdir = Utils.to_hex(Utils.h_list((task.generator.path.abspath(), task.env.variant)))
	task.env.remote_dir = '~%s/wafremote/%s' % (task.env.user, task.env.hdir)
	task.env.local_dir = bld.srcnode.abspath() + '/'

	task.env.remote_dir_variant = '%s/%s/%s' % (task.env.remote_dir, Context.g_module.out, task.env.variant)
	task.env.build_dir = bld.bldnode.abspath()

	ret = task.exec_command(bld.make_mkdir_command(task))
	if ret:
		return ret
	ret = task.exec_command(bld.make_send_command(task))
	if ret:
		return ret
	ret = task.exec_command(bld.make_exec_command(task))
	if ret:
		return ret
	ret = task.exec_command(bld.make_save_command(task))
	if ret:
		return ret