EC2Cloud.java 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459
  1. package hudson.plugins.ec2;
  2. import hudson.model.Computer;
  3. import hudson.model.Descriptor;
  4. import hudson.model.Hudson;
  5. import hudson.model.Label;
  6. import hudson.model.Node;
  7. import hudson.slaves.Cloud;
  8. import hudson.slaves.NodeProvisioner.PlannedNode;
  9. import hudson.util.FormValidation;
  10. import hudson.util.Secret;
  11. import hudson.util.StreamTaskListener;
  12. import java.io.BufferedReader;
  13. import java.io.IOException;
  14. import java.io.StringReader;
  15. import java.io.StringWriter;
  16. import java.net.MalformedURLException;
  17. import java.net.URL;
  18. import java.util.ArrayList;
  19. import java.util.Collection;
  20. import java.util.Collections;
  21. import java.util.Date;
  22. import java.util.List;
  23. import java.util.concurrent.Callable;
  24. import java.util.logging.Level;
  25. import java.util.logging.Logger;
  26. import javax.servlet.ServletException;
  27. import org.kohsuke.stapler.QueryParameter;
  28. import org.kohsuke.stapler.StaplerRequest;
  29. import org.kohsuke.stapler.StaplerResponse;
  30. import com.amazonaws.AmazonClientException;
  31. import com.amazonaws.auth.AWSCredentials;
  32. import com.amazonaws.auth.BasicAWSCredentials;
  33. import com.amazonaws.services.ec2.AmazonEC2;
  34. import com.amazonaws.services.ec2.AmazonEC2Client;
  35. import com.amazonaws.services.ec2.model.CreateKeyPairRequest;
  36. import com.amazonaws.services.ec2.model.Instance;
  37. import com.amazonaws.services.ec2.model.InstanceStateName;
  38. import com.amazonaws.services.ec2.model.InstanceType;
  39. import com.amazonaws.services.ec2.model.KeyPair;
  40. import com.amazonaws.services.ec2.model.KeyPairInfo;
  41. import com.amazonaws.services.ec2.model.Reservation;
  42. import com.amazonaws.services.s3.AmazonS3;
  43. import com.amazonaws.services.s3.AmazonS3Client;
  44. import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest;
  45. /**
  46. * Hudson's view of EC2.
  47. *
  48. * @author Kohsuke Kawaguchi
  49. */
  50. public abstract class EC2Cloud extends Cloud {
  51. public static final String DEFAULT_EC2_HOST = "us-east-1";
  52. public static final String EC2_URL_HOST = "ec2.amazonaws.com";
  53. private final String accessId;
  54. private final Secret secretKey;
  55. private final EC2PrivateKey privateKey;
  56. /**
  57. * Upper bound on how many instances we may provision.
  58. */
  59. public final int instanceCap;
  60. private final List<SlaveTemplate> templates;
  61. private transient KeyPair usableKeyPair;
  62. private transient AmazonEC2 connection;
  63. private static AWSCredentials awsCredentials;
  64. protected EC2Cloud(String id, String accessId, String secretKey, String privateKey, String instanceCapStr, List<SlaveTemplate> templates) {
  65. super(id);
  66. this.accessId = accessId.trim();
  67. this.secretKey = Secret.fromString(secretKey.trim());
  68. this.privateKey = new EC2PrivateKey(privateKey);
  69. if(templates==null) {
  70. this.templates=Collections.emptyList();
  71. } else {
  72. this.templates=templates;
  73. }
  74. if(instanceCapStr.equals("")) {
  75. this.instanceCap = Integer.MAX_VALUE;
  76. } else {
  77. this.instanceCap = Integer.parseInt(instanceCapStr);
  78. }
  79. readResolve(); // set parents
  80. }
  81. public abstract URL getEc2EndpointUrl() throws IOException;
  82. public abstract URL getS3EndpointUrl() throws IOException;
  83. protected Object readResolve() {
  84. for (SlaveTemplate t : templates)
  85. t.parent = this;
  86. return this;
  87. }
  88. public String getAccessId() {
  89. return accessId;
  90. }
  91. public String getSecretKey() {
  92. return secretKey.getEncryptedValue();
  93. }
  94. public EC2PrivateKey getPrivateKey() {
  95. return privateKey;
  96. }
  97. public String getInstanceCapStr() {
  98. if(instanceCap==Integer.MAX_VALUE)
  99. return "";
  100. else
  101. return String.valueOf(instanceCap);
  102. }
  103. public List<SlaveTemplate> getTemplates() {
  104. return Collections.unmodifiableList(templates);
  105. }
  106. public SlaveTemplate getTemplate(String ami) {
  107. for (SlaveTemplate t : templates)
  108. if(t.ami.equals(ami))
  109. return t;
  110. return null;
  111. }
  112. /**
  113. * Gets {@link SlaveTemplate} that has the matching {@link Label}.
  114. */
  115. public SlaveTemplate getTemplate(Label label) {
  116. for (SlaveTemplate t : templates)
  117. if(label == null || label.matches(t.getLabelSet()))
  118. return t;
  119. return null;
  120. }
  121. /**
  122. * Gets the {@link KeyPairInfo} used for the launch.
  123. */
  124. public synchronized KeyPair getKeyPair() throws AmazonClientException, IOException {
  125. if(usableKeyPair==null)
  126. usableKeyPair = privateKey.find(connect());
  127. return usableKeyPair;
  128. }
  129. /**
  130. * Counts the number of instances in EC2 currently running that are using the specifed image.
  131. *
  132. * @param ami If AMI is left null, then all instances are counted.
  133. * <p>
  134. * This includes those instances that may be started outside Hudson.
  135. */
  136. public int countCurrentEC2Slaves(String ami) throws AmazonClientException {
  137. int n=0;
  138. for (Reservation r : connect().describeInstances().getReservations()) {
  139. for (Instance i : r.getInstances()) {
  140. if (ami == null || ami.equals(i.getImageId())) {
  141. InstanceStateName stateName = InstanceStateName.fromValue(i.getState().getName());
  142. if (stateName == InstanceStateName.Pending || stateName == InstanceStateName.Running)
  143. n++;
  144. }
  145. }
  146. }
  147. return n;
  148. }
  149. /**
  150. * Debug command to attach to a running instance.
  151. */
  152. public void doAttach(StaplerRequest req, StaplerResponse rsp, @QueryParameter String id) throws ServletException, IOException, AmazonClientException {
  153. checkPermission(PROVISION);
  154. SlaveTemplate t = getTemplates().get(0);
  155. StringWriter sw = new StringWriter();
  156. StreamTaskListener listener = new StreamTaskListener(sw);
  157. EC2Slave node = t.attach(id,listener);
  158. Hudson.getInstance().addNode(node);
  159. rsp.sendRedirect2(req.getContextPath()+"/computer/"+node.getNodeName());
  160. }
  161. public void doProvision(StaplerRequest req, StaplerResponse rsp, @QueryParameter String ami) throws ServletException, IOException {
  162. checkPermission(PROVISION);
  163. if(ami==null) {
  164. sendError("The 'ami' query parameter is missing",req,rsp);
  165. return;
  166. }
  167. SlaveTemplate t = getTemplate(ami);
  168. if(t==null) {
  169. sendError("No such AMI: "+ami,req,rsp);
  170. return;
  171. }
  172. StringWriter sw = new StringWriter();
  173. StreamTaskListener listener = new StreamTaskListener(sw);
  174. try {
  175. EC2Slave node = t.provision(listener);
  176. Hudson.getInstance().addNode(node);
  177. rsp.sendRedirect2(req.getContextPath()+"/computer/"+node.getNodeName());
  178. } catch (AmazonClientException e) {
  179. e.printStackTrace(listener.error(e.getMessage()));
  180. sendError(sw.toString(),req,rsp);
  181. }
  182. }
  183. @Override
  184. public Collection<PlannedNode> provision(Label label, int excessWorkload) {
  185. try {
  186. final SlaveTemplate t = getTemplate(label);
  187. List<PlannedNode> r = new ArrayList<PlannedNode>();
  188. for( ; excessWorkload>0; excessWorkload-- ) {
  189. if(countCurrentEC2Slaves(null)>=instanceCap) {
  190. LOGGER.log(Level.INFO, "Instance cap reached, not provisioning.");
  191. break; // maxed out
  192. }
  193. int amiCap = t.getInstanceCap();
  194. if (amiCap < countCurrentEC2Slaves(t.ami)) {
  195. LOGGER.log(Level.INFO, "AMI Instance cap reached, not provisioning.");
  196. break; // maxed out
  197. }
  198. r.add(new PlannedNode(t.getDisplayName(),
  199. Computer.threadPoolForRemoting.submit(new Callable<Node>() {
  200. public Node call() throws Exception {
  201. // TODO: record the output somewhere
  202. EC2Slave s = t.provision(new StreamTaskListener(System.out));
  203. Hudson.getInstance().addNode(s);
  204. // EC2 instances may have a long init script. If we declare
  205. // the provisioning complete by returning without the connect
  206. // operation, NodeProvisioner may decide that it still wants
  207. // one more instance, because it sees that (1) all the slaves
  208. // are offline (because it's still being launched) and
  209. // (2) there's no capacity provisioned yet.
  210. //
  211. // deferring the completion of provisioning until the launch
  212. // goes successful prevents this problem.
  213. s.toComputer().connect(false).get();
  214. return s;
  215. }
  216. })
  217. ,t.getNumExecutors()));
  218. }
  219. return r;
  220. } catch (AmazonClientException e) {
  221. LOGGER.log(Level.WARNING,"Failed to count the # of live instances on EC2",e);
  222. return Collections.emptyList();
  223. }
  224. }
  225. @Override
  226. public boolean canProvision(Label label) {
  227. return getTemplate(label)!=null;
  228. }
  229. /**
  230. * Gets the first {@link EC2Cloud} instance configured in the current Hudson, or null if no such thing exists.
  231. */
  232. public static EC2Cloud get() {
  233. return Hudson.getInstance().clouds.get(EC2Cloud.class);
  234. }
  235. /**
  236. * Connects to EC2 and returns {@link AmazonEC2}, which can then be used to communicate with EC2.
  237. */
  238. public synchronized AmazonEC2 connect() throws AmazonClientException {
  239. try {
  240. if (connection == null) {
  241. connection = connect(accessId, secretKey, getEc2EndpointUrl());
  242. }
  243. return connection;
  244. } catch (IOException e) {
  245. throw new AmazonClientException("Failed to retrieve the endpoint",e);
  246. }
  247. }
  248. /***
  249. * Connect to an EC2 instance.
  250. * @return {@link AmazonEC2} client
  251. */
  252. public static AmazonEC2 connect(String accessId, String secretKey, URL endpoint) {
  253. return connect(accessId, Secret.fromString(secretKey), endpoint);
  254. }
  255. /***
  256. * Connect to an EC2 instance.
  257. * @return {@link AmazonEC2} client
  258. */
  259. public static AmazonEC2 connect(String accessId, Secret secretKey, URL endpoint) {
  260. awsCredentials = new BasicAWSCredentials(accessId, Secret.toString(secretKey));
  261. AmazonEC2 client = new AmazonEC2Client(awsCredentials);
  262. client.setEndpoint(endpoint.toString());
  263. return client;
  264. }
  265. /***
  266. * Convert a configured hostname like 'us-east-1' to a FQDN or ip address
  267. */
  268. public static String convertHostName(String ec2HostName) {
  269. if (ec2HostName == null || ec2HostName.length()==0)
  270. ec2HostName = DEFAULT_EC2_HOST;
  271. if (!ec2HostName.contains("."))
  272. ec2HostName = ec2HostName + "." + EC2_URL_HOST;
  273. return ec2HostName;
  274. }
  275. /***
  276. * Convert a user entered string into a port number
  277. * "" -> -1 to indicate default based on SSL setting
  278. */
  279. public static Integer convertPort(String ec2Port) {
  280. if (ec2Port == null || ec2Port.length() == 0)
  281. return -1;
  282. else
  283. return Integer.parseInt(ec2Port);
  284. }
  285. /**
  286. * Computes the presigned URL for the given S3 resource.
  287. *
  288. * @param path
  289. * String like "/bucketName/folder/folder/abc.txt" that represents the resource to request.
  290. */
  291. public URL buildPresignedURL(String path) throws IOException, AmazonClientException {
  292. long expires = System.currentTimeMillis()+60*60*1000;
  293. GeneratePresignedUrlRequest request = new GeneratePresignedUrlRequest(path, Secret.toString(secretKey));
  294. request.setExpiration(new Date(expires));
  295. AmazonS3 s3 = new AmazonS3Client(awsCredentials);
  296. return s3.generatePresignedUrl(request);
  297. }
  298. /* Parse a url or return a sensible error */
  299. public static URL checkEndPoint(String url) throws FormValidation {
  300. try {
  301. return new URL(url);
  302. } catch (MalformedURLException ex) {
  303. throw FormValidation.error("Endpoint URL is not a valid URL");
  304. }
  305. }
  306. public static abstract class DescriptorImpl extends Descriptor<Cloud> {
  307. public InstanceType[] getInstanceTypes() {
  308. return InstanceType.values();
  309. }
  310. public FormValidation doCheckAccessId(@QueryParameter String value) throws IOException, ServletException {
  311. return FormValidation.validateBase64(value,false,false,Messages.EC2Cloud_InvalidAccessId());
  312. }
  313. public FormValidation doCheckSecretKey(@QueryParameter String value) throws IOException, ServletException {
  314. return FormValidation.validateBase64(value,false,false,Messages.EC2Cloud_InvalidSecretKey());
  315. }
  316. public FormValidation doCheckPrivateKey(@QueryParameter String value) throws IOException, ServletException {
  317. boolean hasStart=false,hasEnd=false;
  318. BufferedReader br = new BufferedReader(new StringReader(value));
  319. String line;
  320. while ((line = br.readLine()) != null) {
  321. if (line.equals("-----BEGIN RSA PRIVATE KEY-----"))
  322. hasStart=true;
  323. if (line.equals("-----END RSA PRIVATE KEY-----"))
  324. hasEnd=true;
  325. }
  326. if(!hasStart)
  327. return FormValidation.error("This doesn't look like a private key at all");
  328. if(!hasEnd)
  329. return FormValidation.error("The private key is missing the trailing 'END RSA PRIVATE KEY' marker. Copy&paste error?");
  330. return FormValidation.ok();
  331. }
  332. protected FormValidation doTestConnection( URL ec2endpoint,
  333. String accessId, String secretKey, String privateKey) throws IOException, ServletException {
  334. try {
  335. AmazonEC2 ec2 = connect(accessId, secretKey, ec2endpoint);
  336. ec2.describeInstances();
  337. if(accessId==null)
  338. return FormValidation.error("Access ID is not specified");
  339. if(secretKey==null)
  340. return FormValidation.error("Secret key is not specified");
  341. if(privateKey==null)
  342. return FormValidation.error("Private key is not specified. Click 'Generate Key' to generate one.");
  343. if(privateKey.trim().length()>0) {
  344. // check if this key exists
  345. EC2PrivateKey pk = new EC2PrivateKey(privateKey);
  346. if(pk.find(ec2)==null)
  347. return FormValidation.error("The EC2 key pair private key isn't registered to this EC2 region (fingerprint is "+pk.getFingerprint()+")");
  348. }
  349. return FormValidation.ok(Messages.EC2Cloud_Success());
  350. } catch (AmazonClientException e) {
  351. LOGGER.log(Level.WARNING, "Failed to check EC2 credential",e);
  352. return FormValidation.error(e.getMessage());
  353. }
  354. }
  355. public FormValidation doGenerateKey(StaplerResponse rsp, URL ec2EndpointUrl, String accessId, String secretKey
  356. ) throws IOException, ServletException {
  357. try {
  358. AmazonEC2 ec2 = connect(accessId, secretKey, ec2EndpointUrl);
  359. List<KeyPairInfo> existingKeys = ec2.describeKeyPairs().getKeyPairs();
  360. int n = 0;
  361. while(true) {
  362. boolean found = false;
  363. for (KeyPairInfo k : existingKeys) {
  364. if(k.getKeyName().equals("hudson-"+n))
  365. found=true;
  366. }
  367. if(!found)
  368. break;
  369. n++;
  370. }
  371. CreateKeyPairRequest request = new CreateKeyPairRequest("hudson-" + n);
  372. KeyPair key = ec2.createKeyPair(request).getKeyPair();
  373. rsp.addHeader("script","findPreviousFormItem(button,'privateKey').value='"+key.getKeyMaterial().replace("\n","\\n")+"'");
  374. return FormValidation.ok(Messages.EC2Cloud_Success());
  375. } catch (AmazonClientException e) {
  376. LOGGER.log(Level.WARNING, "Failed to check EC2 credential",e);
  377. return FormValidation.error(e.getMessage());
  378. }
  379. }
  380. }
  381. private static final Logger LOGGER = Logger.getLogger(EC2Cloud.class.getName());
  382. private static boolean isSSL(URL endpoint) {
  383. return endpoint.getProtocol().equals("https");
  384. }
  385. private static int portFromURL(URL endpoint) {
  386. int ec2Port = endpoint.getPort();
  387. if (ec2Port == -1) {
  388. ec2Port = endpoint.getDefaultPort();
  389. }
  390. return ec2Port;
  391. }
  392. }