OneEC2Cloud.java 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555
  1. package hudson.plugins.ec2.one;
  2. import hudson.model.Computer;
  3. import hudson.model.Descriptor;
  4. import hudson.model.Hudson;
  5. import hudson.model.Label;
  6. import hudson.model.Node;
  7. import hudson.plugins.ec2.EC2Cloud;
  8. import hudson.plugins.ec2.EC2PrivateKey;
  9. import hudson.plugins.ec2.SlaveTemplate;
  10. import hudson.slaves.Cloud;
  11. import hudson.slaves.NodeProvisioner.PlannedNode;
  12. import hudson.util.FormValidation;
  13. import hudson.util.Secret;
  14. import hudson.util.StreamTaskListener;
  15. import java.io.BufferedReader;
  16. import java.io.IOException;
  17. import java.io.StringReader;
  18. import java.io.StringWriter;
  19. import java.net.MalformedURLException;
  20. import java.net.URL;
  21. import java.util.ArrayList;
  22. import java.util.Collection;
  23. import java.util.Collections;
  24. import java.util.Date;
  25. import java.util.List;
  26. import java.util.concurrent.Callable;
  27. import java.util.logging.Level;
  28. import java.util.logging.Logger;
  29. import javax.servlet.ServletException;
  30. import org.kohsuke.stapler.QueryParameter;
  31. import org.kohsuke.stapler.StaplerRequest;
  32. import org.kohsuke.stapler.StaplerResponse;
  33. import com.amazonaws.AmazonClientException;
  34. import com.amazonaws.auth.AWSCredentials;
  35. import com.amazonaws.auth.BasicAWSCredentials;
  36. import com.amazonaws.services.ec2.AmazonEC2;
  37. import com.amazonaws.services.ec2.AmazonEC2Client;
  38. import com.amazonaws.services.ec2.model.CreateKeyPairRequest;
  39. import com.amazonaws.services.ec2.model.Instance;
  40. import com.amazonaws.services.ec2.model.InstanceStateName;
  41. import com.amazonaws.services.ec2.model.InstanceType;
  42. import com.amazonaws.services.ec2.model.KeyPair;
  43. import com.amazonaws.services.ec2.model.KeyPairInfo;
  44. import com.amazonaws.services.ec2.model.Reservation;
  45. import com.amazonaws.services.s3.AmazonS3;
  46. import com.amazonaws.services.s3.AmazonS3Client;
  47. import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest;
  48. /**
  49. * Hudson's view of EC2.
  50. *
  51. * @author Kohsuke Kawaguchi
  52. */
  53. public abstract class OneEC2Cloud extends Cloud {
  54. public static final String DEFAULT_EC2_HOST = "us-east-1";
  55. public static final String EC2_URL_HOST = "opennebula";
  56. public static final String EC2_PORT = "4567";
  57. protected final String accessId;
  58. protected final Secret secretKey;
  59. protected final EC2PrivateKey privateKey;
  60. /**
  61. * Upper bound on how many instances we may provision.
  62. */
  63. public final int instanceCap;
  64. protected final List<OneSlaveTemplate> templates;
  65. protected transient KeyPair usableKeyPair;
  66. protected transient AmazonEC2 connection;
  67. protected static AWSCredentials awsCredentials;
  68. protected OneEC2Cloud(final String id, final String accessId,
  69. final String secretKey, final String privateKey,
  70. final String instanceCapStr,
  71. final List<OneSlaveTemplate> templates) {
  72. super(id);
  73. this.accessId = accessId.trim();
  74. this.secretKey = Secret.fromString(secretKey.trim());
  75. this.privateKey =
  76. (null != privateKey) ? new EC2PrivateKey(privateKey) : null;
  77. if (templates == null) {
  78. this.templates = Collections.emptyList();
  79. } else {
  80. this.templates = templates;
  81. }
  82. if (instanceCapStr.equals("")) {
  83. instanceCap = Integer.MAX_VALUE;
  84. } else {
  85. instanceCap = Integer.parseInt(instanceCapStr);
  86. }
  87. readResolve(); // set parents
  88. }
  89. public abstract URL getEc2EndpointUrl() throws IOException;
  90. public abstract URL getS3EndpointUrl() throws IOException;
  91. protected Object readResolve() {
  92. for (OneSlaveTemplate t : templates) {
  93. t.parent = this;
  94. }
  95. return this;
  96. }
  97. public String getAccessId() {
  98. return accessId;
  99. }
  100. public String getSecretKey() {
  101. return secretKey.getEncryptedValue();
  102. }
  103. public EC2PrivateKey getPrivateKey() {
  104. return privateKey;
  105. }
  106. public String getInstanceCapStr() {
  107. if (instanceCap == Integer.MAX_VALUE) {
  108. return "";
  109. } else {
  110. return String.valueOf(instanceCap);
  111. }
  112. }
  113. public List<OneSlaveTemplate> getTemplates() {
  114. return Collections.unmodifiableList(templates);
  115. }
  116. public OneSlaveTemplate getTemplate(final String ami) {
  117. for (OneSlaveTemplate t : templates) {
  118. if (t.ami.equals(ami)) {
  119. return t;
  120. }
  121. }
  122. return null;
  123. }
  124. /**
  125. * Gets {@link SlaveTemplate} that has the matching {@link Label}.
  126. */
  127. public OneSlaveTemplate getTemplate(final Label label) {
  128. for (OneSlaveTemplate t : templates) {
  129. if (label == null || label.matches(t.getLabelSet())) {
  130. return t;
  131. }
  132. }
  133. return null;
  134. }
  135. /**
  136. * Gets the {@link KeyPairInfo} used for the launch.
  137. */
  138. public synchronized KeyPair getKeyPair()
  139. throws AmazonClientException, IOException {
  140. if (usableKeyPair == null && null != privateKey) {
  141. usableKeyPair = privateKey.find(connect());
  142. }
  143. return usableKeyPair;
  144. }
  145. /**
  146. * Counts the number of instances in EC2 currently running that are using
  147. * the specifed image.
  148. *
  149. * @param ami
  150. * If AMI is left null, then all instances are counted.
  151. * <p>
  152. * This includes those instances that may be started outside
  153. * Hudson.
  154. */
  155. public int countCurrentEC2Slaves(final String ami)
  156. throws AmazonClientException {
  157. int n = 0;
  158. for (Reservation r : connect().describeInstances()
  159. .getReservations()) {
  160. for (Instance i : r.getInstances()) {
  161. if (ami == null || ami.equals(i.getImageId())) {
  162. InstanceStateName stateName =
  163. InstanceStateName
  164. .fromValue(i.getState().getName());
  165. if (stateName == InstanceStateName.Pending
  166. || stateName == InstanceStateName.Running) {
  167. n++;
  168. }
  169. }
  170. }
  171. }
  172. return n;
  173. }
  174. /**
  175. * Debug command to attach to a running instance.
  176. */
  177. public void doAttach(final StaplerRequest req,
  178. final StaplerResponse rsp, @QueryParameter final String id)
  179. throws ServletException, IOException, AmazonClientException {
  180. checkPermission(PROVISION);
  181. OneSlaveTemplate t = getTemplates().get(0);
  182. StringWriter sw = new StringWriter();
  183. StreamTaskListener listener = new StreamTaskListener(sw);
  184. OneEC2Slave node = t.attach(id, listener);
  185. Hudson.getInstance().addNode(node);
  186. rsp.sendRedirect2(req.getContextPath() + "/computer/"
  187. + node.getNodeName());
  188. }
  189. public void doProvision(final StaplerRequest req,
  190. final StaplerResponse rsp, @QueryParameter final String ami)
  191. throws ServletException, IOException {
  192. checkPermission(PROVISION);
  193. if (ami == null) {
  194. sendError("The 'ami' query parameter is missing", req, rsp);
  195. return;
  196. }
  197. OneSlaveTemplate t = getTemplate(ami);
  198. if (t == null) {
  199. sendError("No such AMI: " + ami, req, rsp);
  200. return;
  201. }
  202. StringWriter sw = new StringWriter();
  203. StreamTaskListener listener = new StreamTaskListener(sw);
  204. try {
  205. OneEC2Slave node = t.provision(listener);
  206. Hudson.getInstance().addNode(node);
  207. rsp.sendRedirect2(req.getContextPath() + "/computer/"
  208. + node.getNodeName());
  209. } catch (AmazonClientException e) {
  210. e.printStackTrace(listener.error(e.getMessage()));
  211. sendError(sw.toString(), req, rsp);
  212. }
  213. }
  214. @Override
  215. public Collection<PlannedNode> provision(final Label label,
  216. int excessWorkload) {
  217. try {
  218. final OneSlaveTemplate t = getTemplate(label);
  219. List<PlannedNode> r = new ArrayList<PlannedNode>();
  220. for (; excessWorkload > 0; excessWorkload--) {
  221. if (countCurrentEC2Slaves(null) >= instanceCap) {
  222. LOGGER.log(Level.INFO,
  223. "Instance cap reached, not provisioning.");
  224. break; // maxed out
  225. }
  226. int amiCap = t.getInstanceCap();
  227. if (amiCap < countCurrentEC2Slaves(t.ami)) {
  228. LOGGER.log(Level.INFO,
  229. "AMI Instance cap reached, not provisioning.");
  230. break; // maxed out
  231. }
  232. r.add(new PlannedNode(t.getDisplayName(),
  233. Computer.threadPoolForRemoting
  234. .submit(new Callable<Node>() {
  235. public Node call() throws Exception {
  236. // TODO: record the output somewhere
  237. OneEC2Slave s =
  238. t.provision(new StreamTaskListener(
  239. System.out));
  240. Hudson.getInstance().addNode(s);
  241. // EC2 instances may have a long init script. If
  242. // we declare
  243. // the provisioning complete by returning
  244. // without the connect
  245. // operation, NodeProvisioner may decide that it
  246. // still wants
  247. // one more instance, because it sees that (1)
  248. // all the slaves
  249. // are offline (because it's still being
  250. // launched) and
  251. // (2) there's no capacity provisioned yet.
  252. //
  253. // deferring the completion of provisioning
  254. // until the launch
  255. // goes successful prevents this problem.
  256. s.toComputer().connect(false).get();
  257. return s;
  258. }
  259. }), t.getNumExecutors()));
  260. }
  261. return r;
  262. } catch (AmazonClientException e) {
  263. LOGGER.log(Level.WARNING,
  264. "Failed to count the # of live instances on EC2", e);
  265. return Collections.emptyList();
  266. }
  267. }
  268. @Override
  269. public boolean canProvision(final Label label) {
  270. return getTemplate(label) != null;
  271. }
  272. /**
  273. * Gets the first {@link EC2Cloud} instance configured in the current
  274. * Hudson, or null if no such thing exists.
  275. */
  276. public static OneEC2Cloud get() {
  277. return Hudson.getInstance().clouds.get(OneEC2Cloud.class);
  278. }
  279. /**
  280. * Connects to EC2 and returns {@link AmazonEC2}, which can then be used to
  281. * communicate with EC2.
  282. */
  283. public synchronized AmazonEC2 connect() throws AmazonClientException {
  284. try {
  285. if (connection == null) {
  286. connection =
  287. connect(accessId, secretKey, getEc2EndpointUrl());
  288. }
  289. return connection;
  290. } catch (IOException e) {
  291. throw new AmazonClientException(
  292. "Failed to retrieve the endpoint", e);
  293. }
  294. }
  295. /***
  296. * Connect to an EC2 instance.
  297. *
  298. * @return {@link AmazonEC2} client
  299. */
  300. public static AmazonEC2 connect(final String accessId,
  301. final String secretKey, final URL endpoint) {
  302. return connect(accessId, Secret.fromString(secretKey), endpoint);
  303. }
  304. /***
  305. * Connect to an EC2 instance.
  306. *
  307. * @return {@link AmazonEC2} client
  308. */
  309. public static AmazonEC2 connect(final String accessId,
  310. final Secret secretKey, final URL endpoint) {
  311. awsCredentials =
  312. new BasicAWSCredentials(accessId, Secret.toString(secretKey));
  313. AmazonEC2 client = new AmazonEC2Client(awsCredentials);
  314. client.setEndpoint(endpoint.toString());
  315. return client;
  316. }
  317. /***
  318. * Convert a configured hostname like 'us-east-1' to a FQDN or ip address
  319. */
  320. public static String convertHostName(String ec2HostName) {
  321. if (ec2HostName == null || ec2HostName.length() == 0) {
  322. ec2HostName = DEFAULT_EC2_HOST;
  323. }
  324. if (!ec2HostName.contains(".")) {
  325. ec2HostName = ec2HostName + "." + EC2_URL_HOST;
  326. }
  327. return ec2HostName;
  328. }
  329. /***
  330. * Convert a user entered string into a port number "" -> -1 to indicate
  331. * default based on SSL setting
  332. */
  333. public static Integer convertPort(final String ec2Port) {
  334. if (ec2Port == null || ec2Port.length() == 0) {
  335. return -1;
  336. } else {
  337. return Integer.parseInt(ec2Port);
  338. }
  339. }
  340. /**
  341. * Computes the presigned URL for the given S3 resource.
  342. *
  343. * @param path
  344. * String like "/bucketName/folder/folder/abc.txt" that
  345. * represents the resource to request.
  346. */
  347. public URL buildPresignedURL(final String path)
  348. throws IOException, AmazonClientException {
  349. long expires = System.currentTimeMillis() + 60 * 60 * 1000;
  350. GeneratePresignedUrlRequest request =
  351. new GeneratePresignedUrlRequest(path,
  352. Secret.toString(secretKey));
  353. request.setExpiration(new Date(expires));
  354. AmazonS3 s3 = new AmazonS3Client(awsCredentials);
  355. return s3.generatePresignedUrl(request);
  356. }
  357. /* Parse a url or return a sensible error */
  358. public static URL checkEndPoint(final String url)
  359. throws FormValidation {
  360. try {
  361. return new URL(url);
  362. } catch (MalformedURLException ex) {
  363. throw FormValidation.error("Endpoint URL is not a valid URL");
  364. }
  365. }
  366. public static abstract class DescriptorImpl extends Descriptor<Cloud> {
  367. public InstanceType[] getInstanceTypes() {
  368. return InstanceType.values();
  369. }
  370. public FormValidation doCheckAccessId(
  371. @QueryParameter final String value)
  372. throws IOException, ServletException {
  373. return FormValidation.validateBase64(value, false, false,
  374. Messages.EC2Cloud_InvalidAccessId());
  375. }
  376. public FormValidation doCheckSecretKey(
  377. @QueryParameter final String value)
  378. throws IOException, ServletException {
  379. return FormValidation.validateBase64(value, false, false,
  380. Messages.EC2Cloud_InvalidSecretKey());
  381. }
  382. public FormValidation doCheckPrivateKey(
  383. @QueryParameter final String value)
  384. throws IOException, ServletException {
  385. boolean hasStart = false, hasEnd = false;
  386. BufferedReader br =
  387. new BufferedReader(new StringReader(value));
  388. String line;
  389. while ((line = br.readLine()) != null) {
  390. if (line.equals("-----BEGIN RSA PRIVATE KEY-----")) {
  391. hasStart = true;
  392. }
  393. if (line.equals("-----END RSA PRIVATE KEY-----")) {
  394. hasEnd = true;
  395. }
  396. }
  397. if (!hasStart) {
  398. return FormValidation
  399. .error("This doesn't look like a private key at all");
  400. }
  401. if (!hasEnd) {
  402. return FormValidation
  403. .error("The private key is missing the trailing 'END RSA PRIVATE KEY' marker. Copy&paste error?");
  404. }
  405. return FormValidation.ok();
  406. }
  407. protected FormValidation doTestConnection(final URL ec2endpoint,
  408. final String accessId, final String secretKey,
  409. final String privateKey)
  410. throws IOException, ServletException {
  411. try {
  412. AmazonEC2 ec2 = connect(accessId, secretKey, ec2endpoint);
  413. ec2.describeInstances();
  414. if (accessId == null) {
  415. return FormValidation
  416. .error("Access ID is not specified");
  417. }
  418. if (secretKey == null) {
  419. return FormValidation
  420. .error("Secret key is not specified");
  421. }
  422. if (null != privateKey && privateKey.trim().length() > 0) {
  423. // check if this key exists
  424. EC2PrivateKey pk = new EC2PrivateKey(privateKey);
  425. if (pk.find(ec2) == null) {
  426. return FormValidation
  427. .error("The EC2 key pair private key isn't registered to this EC2 region (fingerprint is "
  428. + pk.getFingerprint() + ")");
  429. }
  430. }
  431. return FormValidation.ok(Messages.EC2Cloud_Success());
  432. } catch (AmazonClientException e) {
  433. LOGGER.log(Level.WARNING,
  434. "Failed to check EC2 credential", e);
  435. return FormValidation.error(e.getMessage());
  436. }
  437. }
  438. public FormValidation doGenerateKey(final StaplerResponse rsp,
  439. final URL ec2EndpointUrl, final String accessId,
  440. final String secretKey)
  441. throws IOException, ServletException {
  442. try {
  443. AmazonEC2 ec2 =
  444. connect(accessId, secretKey, ec2EndpointUrl);
  445. List<KeyPairInfo> existingKeys =
  446. ec2.describeKeyPairs().getKeyPairs();
  447. int n = 0;
  448. while (true) {
  449. boolean found = false;
  450. for (KeyPairInfo k : existingKeys) {
  451. if (k.getKeyName().equals("hudson-" + n)) {
  452. found = true;
  453. }
  454. }
  455. if (!found) {
  456. break;
  457. }
  458. n++;
  459. }
  460. CreateKeyPairRequest request =
  461. new CreateKeyPairRequest("hudson-" + n);
  462. KeyPair key = ec2.createKeyPair(request).getKeyPair();
  463. rsp.addHeader("script",
  464. "findPreviousFormItem(button,'privateKey').value='"
  465. + key.getKeyMaterial().replace("\n", "\\n") + "'");
  466. return FormValidation.ok(Messages.EC2Cloud_Success());
  467. } catch (AmazonClientException e) {
  468. LOGGER.log(Level.WARNING,
  469. "Failed to check EC2 credential", e);
  470. return FormValidation.error(e.getMessage());
  471. }
  472. }
  473. }
  474. private static final Logger LOGGER = Logger
  475. .getLogger(OneEC2Cloud.class.getName());
  476. protected static boolean isSSL(final URL endpoint) {
  477. return endpoint.getProtocol().equals("https");
  478. }
  479. protected static int portFromURL(final URL endpoint) {
  480. int ec2Port = endpoint.getPort();
  481. if (ec2Port == -1) {
  482. ec2Port = endpoint.getDefaultPort();
  483. }
  484. return ec2Port;
  485. }
  486. }